Konigle

November 26, 2024

Robots.txt for superuser.com

        User-Agent: *
Disallow: /posts/
Disallow: /posts?
Disallow: /amzn/click/
Disallow: /questions/ask/
Disallow: /questions/ask?
Disallow: /search/
Disallow: /search?
Disallow: /feeds/
Disallow: /feeds?
Disallow: /users/login/
Disallow: /users/login?
Disallow: /users/logout/
Disallow: /users/logout?
Disallow: /users/filter/
Disallow: /users/filter?
Disallow: /users/signup
Disallow: /users/signup/
Disallow: /users/signup?
Disallow: /users/authenticate/
Disallow: /users/authenticate?
Disallow: /users/oauth/*
Disallow: /users/flag-summary/
Disallow: /users/flair/
Disallow: /users/flair?
Disallow: /users/activity/
Disallow: /users/activity/?
Disallow: /users/stats/
Disallow: /users/*?tab=accounts
Disallow: /users/*?tab=activity
Disallow: /users/rep/show
Disallow: /users/rep/show?
Disallow: /unanswered/
Disallow: /unanswered?
Disallow: /u/
Disallow: /messages/
Disallow: /api/*
Disallow: /review/*
Disallow: /*/ivc/*
Disallow: /*?lastactivity
Disallow: /users/login/global/request/
Disallow: /users/login/global/request?
Disallow: /questions/*answertab=*
Disallow: /questions/tagged/*+*
Disallow: /questions/tagged/*%20*
Disallow: /questions/*/answer/submit
Disallow: /tags/*+*
Disallow: /tags/*%20*
Disallow: /suggested-edits/
Disallow: /suggested-edits?
Disallow: /ajax/
Disallow: /plugins/
Disallow: /error
Disallow: /gps/*
Disallow: /10m
Disallow: /jobs/cv/sign-up-and-create
Disallow: /jobs/n/*
Disallow: /jobs/a/*
Disallow: /jobs/apply/*
Disallow: /jobs/companies/n/*
Disallow: /jobs/companies/a/*
Disallow: /jobs/*/more-jobs-count
Disallow: /jobs/email-job
Disallow: /emails/*
Disallow: /_/*
Allow: /
User-agent: Yahoo Pipes 1.0
Disallow: /
User-agent: 008
Disallow: /
User-agent: voltron
Disallow: /
User-agent: Bytespider
Disallow: /
User-Agent: PerplexityBot
Disallow: /
User-agent: meta-externalagent
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: cohere-ai
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: Omgili
Disallow: /
User-agent: magpie-crawler
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: Twitterbot
Disallow: /
User-agent: peer39_crawler
Disallow: /
User-agent: scrapy
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: YouBot
Disallow: /
User-agent: AiZBot
Disallow: /
User-agent: AiZBot-Dolma
Disallow: /
User-agent: Googlebot-Image
Disallow: /*/ivc/*
Disallow: /users/flair/
Disallow: /jobs/n/*
Disallow: /jobs/a/*
Disallow: /jobs/companies/n/*
Disallow: /jobs/companies/a/*
Sitemap: https://superuser.com/sitemap.xml

Keep Reading