# Block Muck Rack and common commercial data scrapers
User-agent: MuckRackBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Diffbot
Disallow: /

# General rules for regular search engines (Google, Bing, etc.)
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# ALLOW all search crawlers to access legacy paths so they hit the 410 walls
Allow: /zh-CN/
Allow: /zh-TW/
Allow: /ja/
Allow: /es/
Allow: /de/
Allow: /it/
Allow: /sv/
Allow: /bg/
Allow: /fr/
Allow: /nl/
Allow: /pl/
Allow: /pt/
Allow: /ru/
2026 Allow: /tr/
Allow: /archives/
Allow: *amp*

# Active Live Sitemaps for your English Content
Sitemap: https://internationalsupermarketnews.com