# ---------------------------
User-agent: *
Disallow: /wp-admin/
User-agent: ia_archiver
Disallow: /
# ---------------------------
# Block CommonCrawl
User-agent: CCBot
Disallow: /
# ---------------------------
# Block OpenAI
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
# ---------------------------
# Block Bard / Gemini crawler
User-agent: Google-Extended
Disallow: /
# ---------------------------
# Block Perplexity AI
User-agent: PerplexityBot
Disallow: /
User-agent: Amazonbot
Disallow: /
# ---------------------------
# Block Anthropic AI
User-agent: ClaudeBot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: Claude-Web
Disallow: /
# ---------------------------
# Block Webz.io
User-agent: Omgili
Disallow: /
User-agent: Omgilibot
Disallow: /
# ---------------------------
# Block Meta
User-Agent: FacebookBot
Disallow: /
# ---------------------------
# Block Apple
User-agent: Applebot-Extended
Disallow: /
# ---------------------------
# Block ByteDance
User-agent: Bytespider
Disallow: /
# ---------------------------
# Block Diffbot
User-agent: Diffbot
Disallow: /
# ---------------------------
# Block Imagesift by Hive
User-agent: ImagesiftBot
Disallow: /
# ---------------------------
# Block YouBot
User-agent: YouBot
Disallow: /
# ---------------------------
# Block cohere
User-agent: cohere-ai
Disallow: /

User-agent: facebookexternalhit
Allow: /

# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow:

Sitemap: https://www.crikey.com.au/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK