# ----------------------------------
# robots.txt for rsd.net
# Last updated: 2026-06-04
# ----------------------------------
#
# Strategy: let crawlers (search + AI) index the real pages — product,
# category and brand/vendor pages — but keep them OUT of the faceted-filter
# URL explosion that spikes CPU and creates duplicate/thin pages.
#
#   Allowed:    /search/?vendno=NNN  (brand pages)
#               /categories/c1[/c2[/c3]]  (clean, path-based category pages)
#               /item/...  (product pages)
#   Disallowed: any /search/ URL with a filter param (starts with "_")
#               any /search/ keyword-results URL
#               any /categories/ URL with a query string (filters/paging/sort)
#
# Discovery is handled by the XML sitemap, so crawlers never need to walk the
# filter combinations to find content.

# -------------------------------------------------
# GLOBAL RULES (APPLY TO ALL BOTS UNLESS OVERRIDDEN)
# -------------------------------------------------
User-agent: *
Disallow: /checkout/
Disallow: /account/
Disallow: /search/item/
Disallow: /search/item.php
Disallow: /search.php
Disallow: /search/*keywords
Disallow: /search/*_
Disallow: /categories/*?
Disallow: /api/
Allow: /

# ------------------------------------
# AI CRAWLERS — WELCOME (same filter protection as everyone)
# ------------------------------------
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Meta-ExternalAgent
User-agent: CCBot
User-agent: cohere-ai
Disallow: /checkout/
Disallow: /account/
Disallow: /search/*keywords
Disallow: /search/*_
Disallow: /categories/*?
Disallow: /api/
Allow: /

# ------------------------------------
# BAD BOTS — BLOCK ENTIRE WEBSITE
# ------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: CensysInspect
Disallow: /

User-agent: python-requests
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: Go-http-client
Disallow: /

User-agent: axios
Disallow: /

User-agent: node-fetch
Disallow: /

User-agent: Java/
Disallow: /

User-agent: libwww-perl
Disallow: /

User-agent: ZGrab
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: PetalBot
Disallow: /

# ------------------------------------
# SITEMAP
# ------------------------------------
Sitemap: https://www.rsd.net/sitemap.xml