### =========================
### AI / Training / Scraper Bots (BLOCK)
### =========================
User-agent: magpie-crawler
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: CCBot
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: cohere-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: bytespider
User-agent: mistralai
User-agent: cohere
User-agent: ai2bot
User-agent: omgili
User-agent: diffbot
User-agent: kangaroo
User-agent: img2dataset
User-agent: Applebot-Extended
User-agent: amazon-qbusiness
User-agent: proximic
User-agent: Greppr
User-agent: TikTokSpider
User-agent: YandexBot
User-agent: Yandex
User-agent: SeznamBot
User-agent: PetalBot
Disallow: /

### =========================
### Platform / Discovery / SEO Tools (THROTTLE + PARTIAL BLOCK)
### =========================
User-agent: Bingbot
User-agent: SemrushBot
User-agent: youbot
User-agent: amazonbot
User-agent: meta-externalagent
User-agent: meta-webindexer
User-agent: Baiduspider
User-agent: Baiduspider-image
User-agent: Baiduspider-video
User-agent: Baiduspider-news
User-agent: Baiduspider-favo
User-agent: AhrefsBot
User-agent: OAI-SearchBot
Disallow: /page/
Disallow: /tag/
Disallow: /author/
Disallow: /feed/
Disallow: /trash/
Disallow: /uncategorized/
Disallow: /?*
Crawl-delay: 20

User-agent: facebookexternalhit
Disallow: /page/
Disallow: /tag/
Disallow: /author/
Disallow: /feed/
Disallow: /trash/
Disallow: /uncategorized/
Disallow: /?*
Crawl-delay: 10

User-agent: Googlebot
Disallow: /feed/
Disallow: /?feed=rss2
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /*?rand=
Disallow: /*?s=
Disallow: /search
Disallow: /?search?
Disallow: /*/feed/
Disallow: /*?iab_category
Disallow: /*?product_tag=
Disallow: /*?dt_commerce_tag=
Disallow: /*?brand=
Disallow: /*?event=
Disallow: /*?product=
Disallow: /*?locations=
Disallow: /*?carrier=
Disallow: /*?entity=
Disallow: /*?game_platform=
Disallow: /*?persons=
Disallow: /*?streaming_service=
Disallow: /*?technology=
Disallow: /*?original_series=
Disallow: /tag/
Disallow: /brc/
Disallow: /sponsored/
Disallow: /aop/
Disallow: /aop-uk/
Disallow: /trash/
Disallow: /uncategorized/

User-agent:*
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /*?rand=
Disallow: /*?s=
Disallow: /search
Disallow: /?search?
Disallow: /*/feed/
Disallow: /*?iab_category
Disallow: /*?product_tag=
Disallow: /*?dt_commerce_tag=
Disallow: /*?brand=
Disallow: /*?event=
Disallow: /*?product=
Disallow: /*?locations=
Disallow: /*?carrier=
Disallow: /*?entity=
Disallow: /*?game_platform=
Disallow: /*?persons=
Disallow: /*?streaming_service=
Disallow: /*?technology=
Disallow: /*?original_series=
Disallow: /tag/
Disallow: /sponsored/
Disallow: /aop/
Disallow: /aop-uk/
Disallow: /trash/
Disallow: /uncategorized/
Sitemap: https://www.digitaltrends.com/sitemap-articles-2024-index.xml
Sitemap: https://www.digitaltrends.com/sitemap-articles-2025-index.xml
Sitemap: https://www.digitaltrends.com/sitemap-all-articles-index.xml
Sitemap: https://www.digitaltrends.com/sitemap-google-news-sitemap_1.xml