diff --git a/layouts/robots.txt b/layouts/robots.txt index f12423a4..d00d8e01 100644 --- a/layouts/robots.txt +++ b/layouts/robots.txt @@ -2,12 +2,60 @@ Sitemap: https://brainbaking.com/sitemap.xml User-agent: * Disallow: -User-agent: ChatGPT-User + +# Dark Visitors Robots.txt + +# AI Data Scraper +# https://darkvisitors.com/agents/anthropic-ai + +User-agent: anthropic-ai Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/bytespider + +User-agent: Bytespider +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/ccbot + +User-agent: CCBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/diffbot + +User-agent: Diffbot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/facebookbot + +User-agent: FacebookBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/google-extended + +User-agent: Google-Extended +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/gptbot + User-agent: GPTBot Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/omgili + +User-agent: omgili +Disallow: / + +User-agent: ChatGPT-User +Disallow: / + User-agent: Mediapartners-Google Disallow: / @@ -17,5 +65,3 @@ Disallow: / User-agent: adidxbot Disallow: / -User-agent: CCBot -Disallow: /