Robot.txt file for: mumsnet.com

      # Mumsnet content is made available for your personal, non-commercial
# use subject to our Terms of Service here:
# https://www.mumsnet.com/i/terms-of-use
# Use of any device, tool, or process designed to data mine or scrape the content
# using automated means is prohibited without prior written permission from
# Mumsnet.  Prohibited uses include but are not limited to:
# (1) text and data mining activities under Art. 4 of the EU Directive on Copyright in
# the Digital Single Market;
# (2) the development of any software, machine learning, artificial intelligence (AI),
# and/or large language models (LLMs);
# (3) creating or providing archived or cached data sets containing our content to others; and/or
# (4) any commercial purposes.
# Contact contactus@mumsnet.com for assistance.

User-agent: *
Disallow: /signin?*
Disallow: /register-user?*
Disallow: /ReportTalkPost*
Disallow: /version-choice*
Disallow: /pm*
Disallow: /session/
Disallow: /login-needed/
Disallow: /ajax/*
Disallow: /s/A
Disallow: /member/
Disallow: /emo/
Disallow: /te/
Disallow: /api/
Disallow: /system/1/
Disallow: /discus/
Disallow: /pies/
Disallow: /talk-pictures/
Disallow: /food/recipe/add-recipe
Disallow: /service/searchlambda/all-search-results*
Disallow: /SearchArch?*
Disallow: /*.pdf
Disallow: /utility/*
Disallow: /talk/*/edit
Disallow: /talk/*flipped=1
Disallow: /search?query=*
Disallow: /login?*
Allow: /.well-known/

# Agent specific disallows
# ------------------------
User-agent: omgilibot
Disallow: /

User-agent: Meltwater
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: 008
Disallow: /

User-agent: voltron
Disallow: /

User-agent: PiplBot
Disallow: /

User-agent: NewsNow
Disallow: /

User-agent: WebVac
Disallow: /

User-agent: WebZip
Disallow: /

User-agent: psbot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-Agent: endeca
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: MoodleBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CrystalSemanticsBot
Disallow: /

User-agent: AwarioRssBot
Disallow: /

User-agent: AwarioSmartBot
Disallow: /

User-agent: trendictionbot
Disallow: /

# Agent specific allows
# ---------------------
User-agent: Googlebot
Allow: /api/v3/talk/topics/*/threads
Allow: /api/v3/discover/subjects/*/threads
Allow: /api/v3/discover/categories/*/threads
Allow: /api/v3/feeds/trending
Allow: /api/v3/feeds/trending/more
Allow: /api/v3/feeds/personal


# Sitemaps
# --------
Sitemap: https://www.mumsnet.com/sitemap/cms/content
Sitemap: https://www.mumsnet.com/sitemap/cms/news
Sitemap: https://www.mumsnet.com/sitemap/talk