# Sitemap
Sitemap: https://www.advertentiex.nl/sitemap/sitemap.xml

# Block everything by default
User-agent: *
Disallow: /

# Allow Google and Bing, minus URLs that are never worth indexing.
# Each crawler only reads its own group, so the rules are repeated.
# - listing URLs with a query string: search (?q=), zone (?z=), price
#   (?min/?max) and attribute filters - all noindex, unbounded in number
# - ?act= on ads: renew/report actions, not content
# - account/admin/utility pages and AJAX endpoints
# /cat/showMenu stays allowed: pages fetch it while rendering (the menu).
# Ad URLs with ?in= stay crawlable on purpose: old indexed ones must be
# recrawled to see the canonical or the 410.

# Allow Google
User-agent: Googlebot
Allow: /cat/showMenu
Disallow: /Advertenties-in-*?
Disallow: /*?act=
Disallow: /*&act=
Disallow: /cat/
Disallow: /rem/
Disallow: /cron
Disallow: /ppipn
Disallow: /status
Disallow: /import
Disallow: /tprint
Disallow: /user-ads-
Disallow: /premium
Disallow: /edit-ad-
Disallow: /delete-ad-
Disallow: /activate-ad-
Disallow: /activate-account-
Disallow: /admin-
Disallow: /bye-user-
Disallow: /my-account
Disallow: /settings
Disallow: /login
Disallow: /logout
Disallow: /new-account
Disallow: /forgot

# Allow Bing (MSN) with crawl delay
User-agent: Bingbot
Crawl-delay: 10
Allow: /cat/showMenu
Disallow: /Advertenties-in-*?
Disallow: /*?act=
Disallow: /*&act=
Disallow: /cat/
Disallow: /rem/
Disallow: /cron
Disallow: /ppipn
Disallow: /status
Disallow: /import
Disallow: /tprint
Disallow: /user-ads-
Disallow: /premium
Disallow: /edit-ad-
Disallow: /delete-ad-
Disallow: /activate-ad-
Disallow: /activate-account-
Disallow: /admin-
Disallow: /bye-user-
Disallow: /my-account
Disallow: /settings
Disallow: /login
Disallow: /logout
Disallow: /new-account
Disallow: /forgot

# disallow Yahoo (Slurp) with crawl delay
User-agent: Slurp
Disallow: /

# Explicitly block known heavy / AI crawlers
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: CriteoBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /