# historicaldata.net robots policy
# AI search crawlers (power AI-search citations): allowed to read public pages and metadata endpoints
User-agent: OAI-SearchBot
Allow: /
Disallow: /file/
Disallow: /dl/
Disallow: /download.html
Disallow: /order2.html
Disallow: /order3.html
Disallow: /t1.html
Disallow: /alpaca.html

User-agent: PerplexityBot
Allow: /
Disallow: /file/
Disallow: /dl/
Disallow: /download.html
Disallow: /order2.html
Disallow: /order3.html
Disallow: /t1.html
Disallow: /alpaca.html

User-agent: Claude-SearchBot
Allow: /
Disallow: /file/
Disallow: /dl/
Disallow: /download.html
Disallow: /order2.html
Disallow: /order3.html
Disallow: /t1.html
Disallow: /alpaca.html

# User-triggered AI fetchers (a user asked the AI to read this site): allowed to read public pages and metadata endpoints
User-agent: ChatGPT-User
Allow: /
Disallow: /file/
Disallow: /dl/
Disallow: /download.html
Disallow: /order2.html
Disallow: /order3.html
Disallow: /t1.html
Disallow: /alpaca.html

User-agent: Perplexity-User
Allow: /
Disallow: /file/
Disallow: /dl/
Disallow: /download.html
Disallow: /order2.html
Disallow: /order3.html
Disallow: /t1.html
Disallow: /alpaca.html

User-agent: Claude-User
Allow: /
Disallow: /file/
Disallow: /dl/
Disallow: /download.html
Disallow: /order2.html
Disallow: /order3.html
Disallow: /t1.html
Disallow: /alpaca.html

# Google/Gemini fetchers: public pages plus lightweight metadata API/MCP are allowed.
# Bulk sample files and signed downloads remain blocked.
User-agent: Googlebot
Allow: /
Allow: /api/
Allow: /mcp
Disallow: /file/
Disallow: /dl/
Disallow: /download.html
Disallow: /order2.html
Disallow: /order3.html
Disallow: /t1.html
Disallow: /alpaca.html

User-agent: Google-Extended
Allow: /
Allow: /api/
Allow: /mcp
Disallow: /file/
Disallow: /dl/
Disallow: /download.html
Disallow: /order2.html
Disallow: /order3.html
Disallow: /t1.html
Disallow: /alpaca.html

User-agent: GoogleOther
Allow: /
Allow: /api/
Allow: /mcp
Disallow: /file/
Disallow: /dl/
Disallow: /download.html
Disallow: /order2.html
Disallow: /order3.html
Disallow: /t1.html
Disallow: /alpaca.html

User-agent: Google-InspectionTool
Allow: /
Allow: /api/
Allow: /mcp
Disallow: /file/
Disallow: /dl/
Disallow: /download.html
Disallow: /order2.html
Disallow: /order3.html
Disallow: /t1.html
Disallow: /alpaca.html

User-agent: Gemini
Allow: /
Allow: /api/
Allow: /mcp
Disallow: /file/
Disallow: /dl/
Disallow: /download.html
Disallow: /order2.html
Disallow: /order3.html
Disallow: /t1.html
Disallow: /alpaca.html

# Everyone else: public content pages allowed; bulk data files, signed downloads, metadata API responses and old test pages are not for crawling
User-agent: *
Allow: /
Disallow: /file/
Disallow: /dl/
Disallow: /download.html
Disallow: /api/
Disallow: /mcp
Disallow: /order2.html
Disallow: /order3.html
Disallow: /t1.html
Disallow: /alpaca.html

# Recommended public AI entry page: https://historicaldata.net/index.html
# Fallback public AI entry page: https://historicaldata.net/public-info.html
Sitemap: https://historicaldata.net/sitemap.xml