From 15d2593adac69d0d74936d5b6a68bbec4605ab80 Mon Sep 17 00:00:00 2001 From: wgroeneveld Date: Thu, 18 Apr 2024 14:41:29 +0200 Subject: [PATCH] update robots.txt for https://darkvisitors.com/docs/set-up-a-robots-txt --- layouts/robots.txt | 52 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/layouts/robots.txt b/layouts/robots.txt index f12423a4..d00d8e01 100644 --- a/layouts/robots.txt +++ b/layouts/robots.txt @@ -2,12 +2,60 @@ Sitemap: https://brainbaking.com/sitemap.xml User-agent: * Disallow: -User-agent: ChatGPT-User + +# Dark Visitors Robots.txt + +# AI Data Scraper +# https://darkvisitors.com/agents/anthropic-ai + +User-agent: anthropic-ai Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/bytespider + +User-agent: Bytespider +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/ccbot + +User-agent: CCBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/diffbot + +User-agent: Diffbot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/facebookbot + +User-agent: FacebookBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/google-extended + +User-agent: Google-Extended +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/gptbot + User-agent: GPTBot Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/omgili + +User-agent: omgili +Disallow: / + +User-agent: ChatGPT-User +Disallow: / + User-agent: Mediapartners-Google Disallow: / @@ -17,5 +65,3 @@ Disallow: / User-agent: adidxbot Disallow: / -User-agent: CCBot -Disallow: /