User-agent: *
Allow: /
Allow: /api/sitemaps/*

# Prevent access to potential vulnerable paths
Disallow: */feed/
Disallow: */rss/
Disallow: */rss2/
Disallow: */atom/
Disallow: */rdf/

# Prevent deep pagination crawling
Disallow: *?page=*
Disallow: *?p=*

# Block potential vulnerability scan paths
Disallow: /readme.html
Disallow: /license.txt
Disallow: *.sql
Disallow: *.SQL

# Block temporary and backup files
Disallow: *.bak
Disallow: *.backup
Disallow: *.tmp
Disallow: /*~

# Block access to development files
Disallow: *.json
Disallow: *.lock
Disallow: *.log

# Block API endpoints except those needed
Disallow: /api/*

# Reduce crawl rate for heavy crawlers
Crawl-delay: 10

# Specific rules for problematic bots
User-agent: PetalBot
Disallow: /

User-agent: SemrushBot
Allow: /
Crawl-delay: 20

User-agent: AhrefsBot
Allow: /
Crawl-delay: 20

User-agent: DotBot
Allow: /
Disallow: /*?page=
Crawl-delay: 15

User-agent: Bytespider
Allow: /
Crawl-delay: 15

# Subdomain-specific sitemap
Host: https://zeeland.sp.nl
Sitemap: https://zeeland.sp.nl/api/sitemaps/index