# soonparts.com - Magento 2 crawler policy
# Search engines may crawl public catalog pages, but dynamic account,
# checkout, search, API, and internal application routes are excluded.

# SEO marketing, backlink, and bulk data crawlers
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: MegaIndex.ru
User-agent: ZoominfoBot
User-agent: Barkrowler
Disallow: /

# AI training and bulk-indexing crawlers
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: meta-externalagent
User-agent: Bytespider
User-agent: Amazonbot
User-agent: Google-Extended
User-agent: Applebot-Extended
Crawl-delay: 10
Allow: /static/
Allow: /media/catalog/product/
Allow: /media/wysiwyg/
Disallow: /*?
Disallow: /*.php$
Disallow: /index.php/
Disallow: /catalogsearch/
Disallow: /*/catalogsearch/
Disallow: /search/
Disallow: /*/search/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /customer/
Disallow: /*/customer/
Disallow: /wishlist/
Disallow: /*/wishlist/
Disallow: /sales/
Disallow: /*/sales/
Disallow: /newsletter/
Disallow: /*/newsletter/
Disallow: /review/
Disallow: /*/review/
Disallow: /sendfriend/
Disallow: /*/sendfriend/
Disallow: /catalog/product_compare/
Disallow: /*/catalog/product_compare/
Disallow: /rest/
Disallow: /*/rest/
Disallow: /graphql
Disallow: /*/graphql
Disallow: /api/
Disallow: /*/api/
Disallow: /paypal/
Disallow: /*/paypal/
Disallow: /stripe/
Disallow: /*/stripe/

# AI search and citation crawlers
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: meta-webindexer
User-agent: Amzn-SearchBot
Crawl-delay: 5
Allow: /static/
Allow: /media/catalog/product/
Allow: /media/wysiwyg/
Disallow: /*?
Disallow: /*.php$
Disallow: /catalogsearch/
Disallow: /*/catalogsearch/
Disallow: /search/
Disallow: /*/search/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /customer/
Disallow: /*/customer/
Disallow: /wishlist/
Disallow: /*/wishlist/
Disallow: /sendfriend/
Disallow: /*/sendfriend/
Disallow: /rest/
Disallow: /*/rest/
Disallow: /graphql
Disallow: /*/graphql
Disallow: /api/
Disallow: /*/api/

# Standard search engines and all other crawlers
User-agent: *
Allow: /static/
Allow: /media/catalog/product/
Allow: /media/wysiwyg/
Allow: /*?p=
Disallow: /*?
Disallow: /*?p=*&*
Disallow: /*.php$
Disallow: /index.php/
Disallow: /catalogsearch/
Disallow: /*/catalogsearch/
Disallow: /search/
Disallow: /*/search/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /customer/
Disallow: /*/customer/
Disallow: /wishlist/
Disallow: /*/wishlist/
Disallow: /sales/
Disallow: /*/sales/
Disallow: /newsletter/
Disallow: /*/newsletter/
Disallow: /review/
Disallow: /*/review/
Disallow: /sendfriend/
Disallow: /*/sendfriend/
Disallow: /catalog/product_compare/
Disallow: /*/catalog/product_compare/
Disallow: /catalog/product/view/
Disallow: /*/catalog/product/view/
Disallow: /catalog/category/view/
Disallow: /*/catalog/category/view/
Disallow: /stores/
Disallow: /*/stores/
Disallow: /directory/currency/
Disallow: /*/directory/currency/
Disallow: /swatches/ajax/
Disallow: /*/swatches/ajax/
Disallow: /rest/
Disallow: /*/rest/
Disallow: /graphql
Disallow: /*/graphql
Disallow: /api/
Disallow: /*/api/
Disallow: /paypal/
Disallow: /*/paypal/
Disallow: /stripe/
Disallow: /*/stripe/
Disallow: /media/customer/
Disallow: /media/import/
Disallow: /media/downloadable/
Disallow: /media/custom_options/
Disallow: /app/
Disallow: /lib/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /vendor/
Disallow: /setup/
Disallow: /update/
Disallow: /errors/
Disallow: /pub/