Robots.txt for stackexchange.com

        User-Agent: *
Disallow: /feeds/
Disallow: /users/login/
Disallow: /users/chat-login/
Disallow: /users/logout/
Disallow: /users/authenticate/
Disallow: /users/flair/
Disallow: /api/
Disallow: /*/ivc/*
Disallow: /*?
Disallow: /leagues/*/week/*/20
Disallow: /leagues/*/month/*/20
Disallow: /leagues/*/quarter/*/20
Disallow: /leagues/*/year/*/20
Disallow: /leagues/*/alltime/*/20
Disallow: /filters/*/*
Disallow: /search/
Disallow: /search?
Allow: /

# for "/*?", refer to http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40360

#
# beware, the sections below WILL NOT INHERIT from the above!
# http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40360
#

User-agent: Spinn3r
Disallow: /

#
# Yahoo Pipes is for feeds not web pages.
#
User-agent: Yahoo Pipes 1.0
Disallow: /

#
# This isn't really an image
#
User-agent: Googlebot-Image
Disallow: /*/ivc/*

#
# KSCrawler - we don't need help from you
#
User-agent: KSCrawler
Disallow: /

#
# ByteSpider is a badly behaving crawler, no thank you!
#
User-agent: Bytespider
Disallow: /

# 
# perplexity.ai FOUND-3020
#
User-Agent: PerplexityBot
Disallow: /

#
# The Meta-ExternalFetcher crawler FOUND-3780
#
User-agent: meta-externalagent
Disallow: /

#
# Applebot FOUND-3780 / FOUND-4082
#
User-agent: Applebot-Extended
Disallow: /
#
# FOUND-3847
#
User-agent: cohere-ai
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: Omgili
Disallow: /
User-agent: magpie-crawler
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: Twitterbot
Disallow: /
User-agent: peer39_crawler
Disallow: /
User-agent: scrapy
Disallow: /
#
# Anthropic AI
#
User-agent: ClaudeBot
Disallow: /
#
# FOUND-3950
#
User-agent: Claude-Web
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: YouBot
Disallow: /
User-agent: AiZBot
Disallow: /
User-agent: AiZBot-Dolma
Disallow: /