User-agent: *
Disallow: /cdn-cgi/
Disallow: /r/
Disallow: /sites/
Disallow: /tag/
Sitemap: https://www.sitesimilar.net/sitemap.xml

User-agent: Yandex
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: 008
Disallow: /

User-Agent: BLEXBot
Disallow: /

User-Agent: AhrefsBot
Disallow: /

User-Agent: Vagabondo
Disallow: /

User-Agent: SEOkicks-Robot
Disallow: /

User-Agent: ia_archiver
Disallow: /

User-Agent: archive.org_bot
Disallow: /

User-Agent: special_archiver
Disallow: /

User-Agent: MJ12bot
Disallow: /

User-Agent: special_archiver
Disallow: /

User-Agent: heritrix
Disallow: /

User-Agent: netEstate NE Crawler
Disallow: /

User-Agent: sistrix
Disallow: /

User-Agent: WBSearchBot
Disallow: /

User-Agent: QuerySeekerSpider
Disallow: /

User-agent: proximic
Disallow: /

User-agent: SiteExplorer
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: adsbot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Neevabot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: GrapeshotCrawler
Disallow: /

User-agent: grapeshot
Disallow: /