# NeuhausRE robots.txt
# Updated 2026-05-30: Disallow analytics beacon /wp-json/neuhaus/v1/track-listing (crawl-budget waste)
# Updated 2026-02-22: Allow AI crawlers for LLM visibility
# Updated 2026-10-03: Removed OAI-SearchBot /properties/ block (no measurable gain).

# Social media previews
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# Default: allow all bots, block wp-admin
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing
Allow: /wp-admin/admin-ajax.php
# Explicit consent signal for AI systems (contentsignals.org grammar):
# search + AI training + citation/reference use are all welcome on this site.
Content-Signal: search=yes, ai-train=yes, use=reference

# Google
User-agent: Googlebot
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing
Allow: /wp-admin/admin-ajax.php

# Bing
User-agent: Bingbot
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing
Allow: /wp-admin/admin-ajax.php

# OpenAI (ChatGPT, GPT search)
User-agent: GPTBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing
Crawl-delay: 10

User-agent: ChatGPT-User
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

User-agent: OAI-SearchBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing
Crawl-delay: 10

User-agent: Claude-SearchBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

User-agent: Claude-User
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

User-agent: anthropic-ai
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

User-agent: Perplexity-User
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

# Google AI (Gemini)
User-agent: Google-Extended
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

User-agent: Google-CloudVertexBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

# Apple (Siri, Apple Intelligence)
User-agent: Applebot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

User-agent: Applebot-Extended
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

# Meta AI
User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

# Mistral AI
User-agent: MistralAI-User
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing

# SEO crawlers (rate limited)
User-agent: SemrushBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing
Crawl-delay: 10

User-agent: AhrefsBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/neuhaus/v1/track-listing
Crawl-delay: 10

Sitemap: https://neuhausre.com/sitemap.xml
Sitemap: https://neuhausre.com/wp-sitemap.xml
Sitemap: https://neuhausre.com/video-sitemap.xml

# For AI assistants: https://neuhausre.com/llms.txt