# ==========================================
# Newswise Robots Policy – Updated 2025-11-03
# ==========================================

Sitemap: https://www.newswise.com/sitemap.xml
Sitemap: https://www.newswise.com/news-sitemap.xml

# ==========================================
# Tier 1  Google Search bots
# ==========================================

User-agent: Googlebot
Allow: /
Disallow: /tracker/c/

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: facebookexternalhit
Allow: /

# ==========================================
# Tier 1  Other major search bots
# ==========================================

User-agent: Applebot
Allow: /
Crawl-delay: 10

User-agent: Bingbot
Allow: /
Crawl-delay: 10

# ==========================================
# Tier 2  SEO crawlers (restricted)
# ==========================================

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: Baiduspider
Crawl-delay: 20

# ==========================================
# Tier 3  Block Bad / Low-Value Crawlers
# ==========================================

User-agent: SputnikBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Majestic
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Baiduspider-video
Disallow: /

User-agent: Baiduspider-image
Disallow: /

# ==========================================
# Tier 4  Block AI Model Training Crawlers
# ==========================================

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Bytespider
Disallow: /

# ==========================================
# Tier 5  Global Rules (apply to ALL bots)
# ==========================================

User-agent: *

# Allowed resources (JS, CSS, Experts pages)
Allow: /javascripts/
Allow: /stylesheets/
Allow: /users/expert/
Allow: /users/expert-list/
Allow: /sitemaps/

# Disallowed internal or duplicate sections
Disallow: /search/
Disallow: /cdn-cgi/l/email-protection
Disallow: /nz/feed_widget/
Disallow: /nz/search_google
Disallow: /reports/
Disallow: /legacy/
Disallow: /images/uploads/
Disallow: /archive/get_wire
Disallow: /nz/clips_view/
Disallow: /archive/list_wires
Disallow: /multimedia/
Disallow: /special-channel/tabarticles/
Disallow: /Users/rotate-image
Disallow: /users/register/
Disallow: /users/login/
Disallow: /users/register
Disallow: /users/login
Disallow: /cdn-cgi/

Crawl-delay: 15