update robots.txt for https://darkvisitors.com/docs/set-up-a-robots-txt
This commit is contained in:
parent
c3ad387eb4
commit
15d2593ada
|
@ -2,12 +2,60 @@ Sitemap: https://brainbaking.com/sitemap.xml
|
||||||
User-agent: *
|
User-agent: *
|
||||||
Disallow:
|
Disallow:
|
||||||
|
|
||||||
User-agent: ChatGPT-User
|
|
||||||
|
# Dark Visitors Robots.txt
|
||||||
|
|
||||||
|
# AI Data Scraper
|
||||||
|
# https://darkvisitors.com/agents/anthropic-ai
|
||||||
|
|
||||||
|
User-agent: anthropic-ai
|
||||||
Disallow: /
|
Disallow: /
|
||||||
|
|
||||||
|
# AI Data Scraper
|
||||||
|
# https://darkvisitors.com/agents/bytespider
|
||||||
|
|
||||||
|
User-agent: Bytespider
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
# AI Data Scraper
|
||||||
|
# https://darkvisitors.com/agents/ccbot
|
||||||
|
|
||||||
|
User-agent: CCBot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
# AI Data Scraper
|
||||||
|
# https://darkvisitors.com/agents/diffbot
|
||||||
|
|
||||||
|
User-agent: Diffbot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
# AI Data Scraper
|
||||||
|
# https://darkvisitors.com/agents/facebookbot
|
||||||
|
|
||||||
|
User-agent: FacebookBot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
# AI Data Scraper
|
||||||
|
# https://darkvisitors.com/agents/google-extended
|
||||||
|
|
||||||
|
User-agent: Google-Extended
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
# AI Data Scraper
|
||||||
|
# https://darkvisitors.com/agents/gptbot
|
||||||
|
|
||||||
User-agent: GPTBot
|
User-agent: GPTBot
|
||||||
Disallow: /
|
Disallow: /
|
||||||
|
|
||||||
|
# AI Data Scraper
|
||||||
|
# https://darkvisitors.com/agents/omgili
|
||||||
|
|
||||||
|
User-agent: omgili
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: ChatGPT-User
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
User-agent: Mediapartners-Google
|
User-agent: Mediapartners-Google
|
||||||
Disallow: /
|
Disallow: /
|
||||||
|
|
||||||
|
@ -17,5 +65,3 @@ Disallow: /
|
||||||
User-agent: adidxbot
|
User-agent: adidxbot
|
||||||
Disallow: /
|
Disallow: /
|
||||||
|
|
||||||
User-agent: CCBot
|
|
||||||
Disallow: /
|
|
||||||
|
|
Loading…
Reference in New Issue