# robots.txt for CryptoAutos.com
# Goal: maximise organic search AND AI-answer-engine discovery of the car catalogue.

# ---------------------------------------------------------------------------
# General web crawlers (Googlebot, Bingbot, etc.)
# ---------------------------------------------------------------------------
User-agent: *
Allow: /

# Genuine admin / private / infrastructure paths (kept out of the index)
Disallow: /admin/
Disallow: /dashboard/
Disallow: /account/
Disallow: /profile-step-1/
Disallow: /profile-step-2/
Disallow: /profile-step-3/
Disallow: /profile-step-4/
Disallow: /secure/
Disallow: /auth/
Disallow: /password-reset/
Disallow: /email-verification/
Disallow: /my-bookings/
Disallow: /api/
Disallow: /private/
# (Disallow: /_next/ removed 2026-09-27 — Googlebot must fetch /_next/static JS+CSS to render pages)

# NOTE: the previous `Disallow: /*?*` blanket-blocked every URL with a query
# string, which silently killed all faceted/long-tail listing pages. It has
# been removed so filtered listing URLs can be crawled and indexed. Path-based
# facets (e.g. /buy-cars/{make}/{model}) remain the canonical, preferred form.

# Allow the money paths explicitly (redundant with Allow: / but documents intent)
Allow: /buy-cars/
Allow: /buy-with-cryptocurrency/
Allow: /cars-for-sale/
Allow: /bikes-for-sale/
Allow: /buy-bikes/
Allow: /mk/
Allow: /partner-dealer/
Allow: /dealers/

# ---------------------------------------------------------------------------
# AI answer-engine / LLM crawlers.
# Business decision: ALLOW. CryptoAutos stock is public and low-moat, and
# surfacing it inside ChatGPT / Perplexity / Google AI Overviews / Claude is a
# free distribution channel that fits a crypto-forward marketplace. Flip any of
# these to `Disallow: /` to opt a given engine out.
# ---------------------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: Applebot-Extended
Allow: /

# ---------------------------------------------------------------------------
# Sitemaps
# ---------------------------------------------------------------------------
Sitemap: https://www.cryptoautos.com/sitemap.xml