Sitemap: https://www.executivegroupfl.com/sitemap-index.xml

# =========================================================
# DEFAULT: deny everything not explicitly allowed below.
# Any crawler not named in this file — known or unknown,
# AI or otherwise — is blocked by default.
# =========================================================
User-agent: *
Disallow: /

# =========================================================
# SEARCH ENGINES — drive organic search traffic
# =========================================================
User-agent: Googlebot
Disallow: /index.php?advanced=1
Disallow: /index.php?quick=1
Disallow: /emails_kvarea/
Disallow: /mobilefirst/
Disallow: /private_kvarea/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /admin/
Allow: /sell
Allow: /areas
Allow: /pages
Allow: /resources
Allow: /index.php?showagency=1
Allow: /index.php?showagent=1
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /
Crawl-delay: 10

User-agent: bingbot
Allow: /
Crawl-delay: 120

# =========================================================
# GOOGLE ADS — required for marketing team's Google Ads
# campaigns (landing page checks, ad quality/relevance scoring)
# =========================================================
User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: AdsBot-Google-Mobile-Apps
Allow: /

# =========================================================
# SOCIAL PREVIEW BOTS — generate link previews when listings
# or pages are shared, driving click-through traffic
# =========================================================
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

# =========================================================
# AI ANSWER / REAL-TIME SEARCH CRAWLERS — fetch content live
# to answer user questions, can cite/link back as a lead source
# =========================================================
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Amazonbot
Allow: /

# =========================================================
# SEO / SITE-AUDIT TOOLS — kept from the original file for
# internal SEO/QA use, not a direct traffic/lead source
# =========================================================
User-agent: PowerMapper
Allow: /
Crawl-delay: 5

User-agent: Screaming Frog SEO Spider
Allow: /

User-agent: SemrushBot
Allow: /

# =========================================================
# COOKIE / PRIVACY COMPLIANCE SCANNERS — kept from the original
# file so consent-banner services can keep scanning the live site
# =========================================================
User-agent: TermlyBot
Allow: /

User-agent: CookieYesbot
Allow: /

User-agent: Osano Privacy Compliance Scanner
Allow: /

# =========================================================
# EXCLUDED (falls under the default Disallow: / above)
# Kept here only as a record of what was deliberately left out
# and why, so it isn't mistaken for an oversight later:
#
#   AI training / bulk-scraping crawlers (no traffic back):
#     GPTBot, CCBot, Bytespider, Google-Extended, anthropic-ai,
#     Applebot-Extended, Diffbot, Omgilibot, Omgili, YouBot
#
#   Ad-serving crawler (AdSense revenue, not campaign traffic —
#   different from AdsBot-Google above, which is allowed):
#     Mediapartners-Google
#
#   Deprecated/inactive:
#     Slurp (Yahoo), msnbot
# =========================================================