# robots.txt for example.com
# Last updated: 2025-07-19

# Allow all Google bots
User-agent: Googlebot
Disallow:

User-agent: Googlebot-Image
Disallow:

User-agent: Googlebot-News
Disallow:

User-agent: Googlebot-Video
Disallow:

User-agent: AdsBot-Google
Disallow:

User-agent: AdsBot-Google-Mobile
Disallow:

# Slow down generic/basic crawlers
User-agent: *
Crawl-delay: 10
Disallow:

# Block known bad crawlers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Yandex
Disallow: 

User-agent: Baiduspider
Disallow: /

User-agent: spbot
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: Seekport
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: trendictionbot
Disallow: /

User-agent: UptimeRobot
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: 

# Sitemap


Sitemap: https://bottleneckcalculator.to/map.xml