# robots.txt unified policy (generated 2026-07-11 19:12:42 UTC) # Existing sitemaps preserved: Sitemap: https://www.monroestreetbooks.com/shop/sitemap/sitemap.xml Sitemap: https://www.monroestreetbooks.com/sitemap.xml # Google family User-agent: Googlebot Crawl-delay: 0 Disallow: /checkout User-agent: Googlebot-Image Crawl-delay: 0 Disallow: /checkout User-agent: Googlebot-News Crawl-delay: 0 Disallow: /checkout User-agent: Googlebot-Video Crawl-delay: 0 Disallow: /checkout User-agent: GoogleOther Crawl-delay: 0 Disallow: /checkout User-agent: Mediapartners-Google Crawl-delay: 0 Disallow: /checkout User-agent: AdsBot-Google Crawl-delay: 0 Disallow: /checkout User-agent: AdsBot-Google-Mobile Crawl-delay: 0 Disallow: /checkout # Bing family User-agent: Bingbot Crawl-delay: 0 Disallow: /checkout User-agent: MSNBot Crawl-delay: 0 Disallow: /checkout User-agent: AdIdxBot Crawl-delay: 0 Disallow: /checkout User-agent: BingPreview Crawl-delay: 0 Disallow: /checkout # Yahoo User-agent: Slurp Crawl-delay: 0 Disallow: /checkout # Default for all other crawlers User-agent: * Crawl-delay: 5 Disallow: /checkout # Note: /checkout is disallowed for all crawlers (order/account pages); nothing else is blocked. # Anthropic / Claude family (explicitly allowed) User-agent: ClaudeBot Allow: / Crawl-delay: 0 Disallow: /checkout User-agent: Claude-Web Allow: / Crawl-delay: 0 Disallow: /checkout User-agent: Claude-User Allow: / Crawl-delay: 0 Disallow: /checkout User-agent: Claude-SearchBot Allow: / Crawl-delay: 0 Disallow: /checkout User-agent: anthropic-ai Allow: / Crawl-delay: 0 Disallow: /checkout # AI answer/search engines that drive referral traffic (explicitly allowed) User-agent: OAI-SearchBot Allow: / Crawl-delay: 0 Disallow: /checkout User-agent: ChatGPT-User Allow: / Crawl-delay: 0 Disallow: /checkout User-agent: PerplexityBot Allow: / Crawl-delay: 0 Disallow: /checkout # Deep search result pages (/search/.../rec/N) only, 2026-10-17 (Ken): PerplexityBot fetched # ~5,300 of them a week and they filled the PHP pool. Items, categories and page 1 of a search stay open. Disallow: /search/*/rec/ User-agent: Perplexity-User Allow: / Crawl-delay: 0 Disallow: /checkout User-agent: Google-Extended Allow: / Crawl-delay: 0 Disallow: /checkout # Training scrapers blocked from catalog search (added 2026-07-28) # Reason: /search pagination exposes a full page list, and each search request # costs a full-catalog stock scan. These bots don't drive referral traffic, so # losing /search costs nothing. Search remains crawlable for Google, Bing and # the AI search engines above, which is what the owner's on-page /search/ links # rely on. /checkout is handled by .htaccess + noindex, not here. User-agent: Amazonbot Crawl-delay: 5 Disallow: /search User-agent: meta-externalagent Crawl-delay: 5 Disallow: /search User-agent: meta-webindexer Crawl-delay: 5 Disallow: /search