# ===========================================================
# InsureDirect.com — robots.txt
# Last updated: August 2026
# Goal: keep admin, login, and thin/duplicate archive pages out
# of every crawler, while giving every crawler, search and AI,
# full access to the real content.
#
# IMPORTANT: a crawler with its own "User-agent:" group ignores
# "User-agent: *" completely, it does not add to it, it replaces
# it. So every crawler listed below shares ONE rule set on
# purpose. Giving any of them a separate group with only
# "Allow: /" and nothing else, the mistake in the previous
# version, hands that crawler full access to everything the
# rules below are meant to keep hidden, including /wp-admin/,
# /wp-login.php, and the two memory-lane pages.
# ===========================================================

User-agent: *
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Bingbot
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: YouBot
User-agent: Amazonbot
User-agent: CCBot
User-agent: cohere-ai
User-agent: Grokbot
User-agent: ia_archiver
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-json/
Disallow: /xmlrpc.php
Disallow: /tag/
Disallow: /category/
Disallow: /author/
Disallow: /blog/page/
Disallow: /2021/
Disallow: /2022/
Disallow: /2023/
Disallow: /2024/
Disallow: /2025/
Disallow: /2026/
Disallow: /?s=
Disallow: /search/
Disallow: /feed/
Disallow: /trackback/
Disallow: /comments/
Disallow: /play-memory-lane/
Disallow: /1960s-memory-lane/
Disallow: /wp-content/plugins/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-content/themes/

# ── SITEMAP ───────────────────────────────────────────────
Sitemap: https://insuredirect.com/sitemap_index.xml