User-agent: *
Allow: /
Allow: /artikel/
Allow: /images/
Allow: /clean-urgent-id/

Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /private/
Disallow: /tmp/

# CSS, JS, dan Images
Allow: *.css
Allow: *.js
Allow: *.webp
Allow: *.jpg
Allow: *.png
Allow: *.ico

# Sitemap
Sitemap: https://www.servicekomputersejakurat.com/sitemap.xml

# Bot-specific directives
User-agent: Googlebot
Allow: /
Allow: /artikel/
Allow: /clean-urgent-id/
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: *.webp
Allow: *.jpg
Allow: *.png
Allow: /images/

User-agent: Googlebot-News
Allow: /artikel/

User-agent: Bingbot
Allow: /
Allow: /artikel/
Crawl-delay: 2

User-agent: Slurp
Allow: /
Crawl-delay: 3

# Social Media Bots
User-agent: FacebookExternalHit
Allow: /
Allow: /artikel/
Crawl-delay: 2

User-agent: Twitterbot
Allow: /
Allow: /artikel/
Crawl-delay: 2

# AMP Specific
User-agent: Googlebot-AMP
Allow: /index-amp.htm
Allow: /artikel/

# Aggressive bots (slow down)
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 5