prettysunflower/kakigoori
1
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Added ai.robots.txt rules for traffic filtering

Browse Source
This commit is contained in:
prettysunflower
2025-03-25 17:41:51 -04:00
parent ed956f5d5d
commit 12f0057cf5
1 changed files with 5 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
kakigoori/traffic_rules.json
View File

@@ -119,6 +119,11 @@
"name": "InternetMeasurement",
"user_agent_regex": "InternetMeasurement",
"action": "DENY"
},
{
"name": "OtherAI (ai.robots.txt)",
"user_agent_regex": "(AI2Bot|Ai2Bot\\-Dolma|Amazonbot|anthropic\\-ai|Applebot|Applebot\\-Extended|Brightbot\\ 1\\.0|Bytespider|CCBot|ChatGPT\\-User|Claude\\-Web|ClaudeBot|cohere\\-ai|cohere\\-training\\-data\\-crawler|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|FriendlyCrawler|Google\\-Extended|GoogleOther|GoogleOther\\-Image|GoogleOther\\-Video|GPTBot|iaskspider/2\\.0|ICC\\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\\ Bot|Meta\\-ExternalAgent|Meta\\-ExternalFetcher|OAI\\-SearchBot|omgili|omgilibot|PanguBot|PerplexityBot|PetalBot|Scrapy|SemrushBot\\-OCOB|SemrushBot\\-SWA|Sidetrade\\ indexer\\ bot|Timpibot|VelenPublicWebCrawler|Webzio\\-Extended|YouBot)",
"action": "DENY"
}
]
}