{ "name": "Who blocks AI crawlers (canaicrawl)", "url": "https://canaicrawl.com/who-blocks-ai", "license": "Free to use; please link back to https://canaicrawl.com/who-blocks-ai.", "sites_source": "Majestic Million (https://majestic.com/reports/majestic-million), CC BY 3.0", "checked": "2026-11-09T20:27:51.154Z", "states": { "found": "robots.txt read", "none": "no robots.txt (4xx): every bot allowed", "refused": "request refused or challenged: unknown", "html": "answered with a web page: unknown", "error": "server error or no answer: unknown" }, "sites": [ { "rank": 1, "domain": "google.com", "state": "found", "status": 200, "robotsTxt": "https://www.google.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/google.com" }, { "rank": 2, "domain": "facebook.com", "state": "found", "status": 200, "robotsTxt": "https://facebook.com/robots.txt", "blocked": [ "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/facebook.com" }, { "rank": 3, "domain": "youtube.com", "state": "found", "status": 200, "robotsTxt": "https://www.youtube.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/youtube.com" }, { "rank": 4, "domain": "instagram.com", "state": "found", "status": 200, "robotsTxt": "https://www.instagram.com/robots.txt", "blocked": [ "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/instagram.com" }, { "rank": 5, "domain": "twitter.com", "state": "found", "status": 200, "robotsTxt": "https://twitter.com/robots.txt", "blocked": [ "DuckDuckBot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/twitter.com" }, { "rank": 6, "domain": "linkedin.com", "state": "found", "status": 200, "robotsTxt": "https://www.linkedin.com/robots.txt", "blocked": [ "PetalBot", "YandexBot", "PerplexityBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "Baiduspider", "OAI-SearchBot", "Claude-SearchBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/linkedin.com" }, { "rank": 7, "domain": "apple.com", "state": "found", "status": 200, "robotsTxt": "https://www.apple.com/robots.txt", "blocked": [ "Baiduspider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/apple.com" }, { "rank": 9, "domain": "microsoft.com", "state": "found", "status": 200, "robotsTxt": "https://www.microsoft.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/microsoft.com" }, { "rank": 10, "domain": "github.com", "state": "found", "status": 200, "robotsTxt": "https://github.com/robots.txt", "blocked": [ "Bytespider" ], "partlyBlocked": [ "Bingbot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "GPTBot", "ClaudeBot", "ChatGPT-User", "Claude-User", "Perplexity-User" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/github.com" }, { "rank": 12, "domain": "wikipedia.org", "state": "found", "status": 200, "robotsTxt": "https://en.wikipedia.org/robots.txt", "blocked": [], "partlyBlocked": [ "SemrushBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/wikipedia.org" }, { "rank": 16, "domain": "wordpress.org", "state": "found", "status": 200, "robotsTxt": "https://wordpress.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/wordpress.org" }, { "rank": 17, "domain": "x.com", "state": "found", "status": 200, "robotsTxt": "https://x.com/robots.txt", "blocked": [ "DuckDuckBot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/x.com" }, { "rank": 22, "domain": "pinterest.com", "state": "found", "status": 200, "robotsTxt": "https://www.pinterest.com/robots.txt", "blocked": [ "PetalBot", "YandexBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "MJ12bot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "Baiduspider", "DuckAssistBot", "AhrefsBot", "SemrushBot", "DotBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/pinterest.com" }, { "rank": 23, "domain": "tiktok.com", "state": "found", "status": 200, "robotsTxt": "https://www.tiktok.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User" ], "partlyBlocked": [ "Bingbot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/tiktok.com" }, { "rank": 26, "domain": "nginx.org", "state": "found", "status": 200, "robotsTxt": "https://nginx.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nginx.org" }, { "rank": 27, "domain": "whatsapp.com", "state": "found", "status": 200, "robotsTxt": "https://www.whatsapp.com/robots.txt", "blocked": [ "PetalBot", "PerplexityBot", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "Applebot-Extended" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/whatsapp.com" }, { "rank": 28, "domain": "vimeo.com", "state": "found", "status": 200, "robotsTxt": "https://vimeo.com/robots.txt", "blocked": [ "Bingbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/vimeo.com" }, { "rank": 29, "domain": "f5.com", "state": "found", "status": 200, "robotsTxt": "https://www.f5.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/f5.com" }, { "rank": 31, "domain": "amazon.com", "state": "found", "status": 200, "robotsTxt": "https://www.amazon.com/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/amazon.com" }, { "rank": 35, "domain": "mozilla.org", "state": "found", "status": 200, "robotsTxt": "https://www.mozilla.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/mozilla.org" }, { "rank": 37, "domain": "europa.eu", "state": "found", "status": 200, "robotsTxt": "https://europa.eu/robots.txt", "blocked": [], "partlyBlocked": [ "Bingbot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/europa.eu" }, { "rank": 39, "domain": "adobe.com", "state": "found", "status": 200, "robotsTxt": "https://www.adobe.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/adobe.com" }, { "rank": 40, "domain": "wordpress.com", "state": "found", "status": 200, "robotsTxt": "https://wordpress.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/wordpress.com" }, { "rank": 44, "domain": "apache.org", "state": "found", "status": 200, "robotsTxt": "https://apache.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/apache.org" }, { "rank": 48, "domain": "reddit.com", "state": "found", "status": 200, "robotsTxt": "https://www.reddit.com/robots.txt", "blocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/reddit.com" }, { "rank": 50, "domain": "nih.gov", "state": "found", "status": 200, "robotsTxt": "https://www.nih.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nih.gov" }, { "rank": 51, "domain": "yahoo.com", "state": "found", "status": 200, "robotsTxt": "https://www.yahoo.com/robots.txt", "blocked": [ "PetalBot", "PerplexityBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Bytespider", "ChatGPT-User" ], "partlyBlocked": [ "OAI-SearchBot", "Claude-SearchBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/yahoo.com" }, { "rank": 52, "domain": "spotify.com", "state": "found", "status": 200, "robotsTxt": "https://www.spotify.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/spotify.com" }, { "rank": 53, "domain": "qq.com", "state": "found", "status": 200, "robotsTxt": "https://www.qq.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/qq.com" }, { "rank": 58, "domain": "godaddy.com", "state": "refused", "status": 403, "robotsTxt": "https://www.godaddy.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/godaddy.com" }, { "rank": 60, "domain": "tumblr.com", "state": "found", "status": 200, "robotsTxt": "https://www.tumblr.com/robots.txt", "blocked": [ "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/tumblr.com" }, { "rank": 61, "domain": "archive.org", "state": "found", "status": 200, "robotsTxt": "https://archive.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/archive.org" }, { "rank": 62, "domain": "macromedia.com", "state": "found", "status": 200, "robotsTxt": "https://www.adobe.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/macromedia.com" }, { "rank": 63, "domain": "creativecommons.org", "state": "found", "status": 200, "robotsTxt": "https://creativecommons.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/creativecommons.org" }, { "rank": 67, "domain": "w3.org", "state": "found", "status": 200, "robotsTxt": "https://www.w3.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/w3.org" }, { "rank": 68, "domain": "flickr.com", "state": "found", "status": 200, "robotsTxt": "https://flickr.com/robots.txt", "blocked": [ "YandexBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "PetalBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "ChatGPT-User", "Claude-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/flickr.com" }, { "rank": 69, "domain": "nytimes.com", "state": "found", "status": 200, "robotsTxt": "https://www.nytimes.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "Meta-ExternalFetcher" ], "partlyBlocked": [ "Googlebot", "Applebot", "Amazonbot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/nytimes.com" }, { "rank": 70, "domain": "baidu.com", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/baidu.com" }, { "rank": 71, "domain": "doi.org", "state": "none", "status": 404, "robotsTxt": "https://www.doi.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/doi.org" }, { "rank": 73, "domain": "medium.com", "state": "found", "status": 200, "robotsTxt": "https://medium.com/robots.txt", "blocked": [ "Amazonbot", "GPTBot", "ClaudeBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/medium.com" }, { "rank": 76, "domain": "namecheap.com", "state": "found", "status": 200, "robotsTxt": "https://www.namecheap.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/namecheap.com" }, { "rank": 77, "domain": "soundcloud.com", "state": "found", "status": 200, "robotsTxt": "https://soundcloud.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/soundcloud.com" }, { "rank": 78, "domain": "forbes.com", "state": "found", "status": 200, "robotsTxt": "https://www.forbes.com/robots.txt", "blocked": [ "PerplexityBot", "Amazonbot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/forbes.com" }, { "rank": 81, "domain": "sciencedirect.com", "state": "refused", "status": 403, "robotsTxt": "https://www.sciencedirect.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/sciencedirect.com" }, { "rank": 82, "domain": "theguardian.com", "state": "found", "status": 200, "robotsTxt": "https://www.theguardian.com/robots.txt", "blocked": [ "PetalBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Claude-User", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/theguardian.com" }, { "rank": 83, "domain": "zoom.us", "state": "found", "status": 200, "robotsTxt": "https://zoom.us/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/zoom.us" }, { "rank": 85, "domain": "sourceforge.net", "state": "found", "status": 200, "robotsTxt": "https://sourceforge.net/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/sourceforge.net" }, { "rank": 87, "domain": "paypal.com", "state": "found", "status": 200, "robotsTxt": "https://www.paypal.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/paypal.com" }, { "rank": 88, "domain": "cpanel.net", "state": "found", "status": 200, "robotsTxt": "https://www.cpanel.net/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cpanel.net" }, { "rank": 89, "domain": "who.int", "state": "found", "status": 200, "robotsTxt": "https://www.who.int/robots.txt", "blocked": [ "CCBot", "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/who.int" }, { "rank": 90, "domain": "shopify.com", "state": "found", "status": 200, "robotsTxt": "https://shopify.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/shopify.com" }, { "rank": 91, "domain": "oracle.com", "state": "refused", "status": 403, "robotsTxt": "https://www.oracle.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/oracle.com" }, { "rank": 93, "domain": "opera.com", "state": "found", "status": 200, "robotsTxt": "https://www.opera.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/opera.com" }, { "rank": 94, "domain": "dropbox.com", "state": "found", "status": 200, "robotsTxt": "https://www.dropbox.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/dropbox.com" }, { "rank": 95, "domain": "bbc.com", "state": "found", "status": 200, "robotsTxt": "https://www.bbc.com/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Perplexity-User", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bbc.com" }, { "rank": 97, "domain": "vk.com", "state": "found", "status": 200, "robotsTxt": "https://vk.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/vk.com" }, { "rank": 98, "domain": "cnn.com", "state": "refused", "status": 403, "robotsTxt": "https://cnn.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/cnn.com" }, { "rank": 99, "domain": "researchgate.net", "state": "found", "status": 200, "robotsTxt": "https://www.researchgate.net/robots.txt", "blocked": [ "Applebot-Extended", "GoogleOther" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/researchgate.net" }, { "rank": 101, "domain": "reuters.com", "state": "found", "status": 200, "robotsTxt": "https://www.reuters.com/robots.txt", "blocked": [ "DuckDuckBot", "PetalBot", "YandexBot", "Baiduspider", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "OAI-SearchBot", "GoogleOther", "ChatGPT-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/reuters.com" }, { "rank": 102, "domain": "nature.com", "state": "found", "status": 200, "robotsTxt": "https://www.nature.com/robots.txt", "blocked": [ "PerplexityBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nature.com" }, { "rank": 103, "domain": "discord.com", "state": "found", "status": 200, "robotsTxt": "https://discord.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/discord.com" }, { "rank": 104, "domain": "launchpad.net", "state": "found", "status": 200, "robotsTxt": "https://launchpad.net/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/launchpad.net" }, { "rank": 105, "domain": "mit.edu", "state": "found", "status": 200, "robotsTxt": "https://web.mit.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/mit.edu" }, { "rank": 106, "domain": "wikimedia.org", "state": "none", "status": 404, "robotsTxt": "https://www.wikimedia.org//robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/wikimedia.org" }, { "rank": 108, "domain": "ibm.com", "state": "found", "status": 200, "robotsTxt": "https://www.ibm.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ibm.com" }, { "rank": 109, "domain": "bbc.co.uk", "state": "found", "status": 200, "robotsTxt": "https://www.bbc.co.uk/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Perplexity-User", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bbc.co.uk" }, { "rank": 110, "domain": "springer.com", "state": "found", "status": 200, "robotsTxt": "https://www.springer.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/springer.com" }, { "rank": 111, "domain": "bing.com", "state": "found", "status": 200, "robotsTxt": "https://www.bing.com/robots.txt?toWww=1&redig=6FB069C81A8A44A287147C1481EB9FBB", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bing.com" }, { "rank": 113, "domain": "php.net", "state": "found", "status": 200, "robotsTxt": "https://www.php.net/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/php.net" }, { "rank": 115, "domain": "harvard.edu", "state": "found", "status": 200, "robotsTxt": "https://www.harvard.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/harvard.edu" }, { "rank": 116, "domain": "wiley.com", "state": "found", "status": 200, "robotsTxt": "https://www.wiley.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "AhrefsBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/wiley.com" }, { "rank": 117, "domain": "weibo.com", "state": "found", "status": 200, "robotsTxt": "https://weibo.com/robots.txt", "blocked": [ "DuckDuckBot", "PetalBot", "YandexBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/weibo.com" }, { "rank": 119, "domain": "gnu.org", "state": "found", "status": 200, "robotsTxt": "https://www.gnu.org/robots.txt", "blocked": [ "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gnu.org" }, { "rank": 121, "domain": "cdc.gov", "state": "found", "status": 200, "robotsTxt": "https://www.cdc.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cdc.gov" }, { "rank": 122, "domain": "un.org", "state": "found", "status": 200, "robotsTxt": "https://www.un.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/un.org" }, { "rank": 123, "domain": "debian.org", "state": "none", "status": 404, "robotsTxt": "https://www.debian.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/debian.org" }, { "rank": 124, "domain": "linktr.ee", "state": "found", "status": 200, "robotsTxt": "https://linktr.ee/robots.txt", "blocked": [ "DuckDuckBot", "PetalBot", "YandexBot", "YouBot", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "GPTBot", "ClaudeBot", "CCBot", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/linktr.ee" }, { "rank": 125, "domain": "issuu.com", "state": "found", "status": 200, "robotsTxt": "https://issuu.com/robots.txt", "blocked": [ "PetalBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/issuu.com" }, { "rank": 127, "domain": "imdb.com", "state": "found", "status": 200, "robotsTxt": "https://www.imdb.com/robots.txt", "blocked": [ "PetalBot", "YandexBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "Baiduspider", "DuckAssistBot", "Amazonbot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/imdb.com" }, { "rank": 128, "domain": "plesk.com", "state": "found", "status": 200, "robotsTxt": "https://www.plesk.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/plesk.com" }, { "rank": 129, "domain": "weebly.com", "state": "found", "status": 200, "robotsTxt": "https://www.weebly.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/weebly.com" }, { "rank": 130, "domain": "expireddomains.com", "state": "found", "status": 200, "robotsTxt": "https://expireddomains.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/expireddomains.com" }, { "rank": 131, "domain": "stanford.edu", "state": "found", "status": 200, "robotsTxt": "https://www.stanford.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/stanford.edu" }, { "rank": 133, "domain": "openai.com", "state": "found", "status": 200, "robotsTxt": "https://openai.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/openai.com" }, { "rank": 135, "domain": "www.gov.uk", "state": "found", "status": 200, "robotsTxt": "https://www.gov.uk/robots.txt", "blocked": [ "AhrefsBot" ], "partlyBlocked": [ "Meta-ExternalAgent" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/www.gov.uk" }, { "rank": 136, "domain": "miit.gov.cn", "state": "none", "status": 404, "robotsTxt": "https://www.miit.gov.cn/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/miit.gov.cn" }, { "rank": 138, "domain": "bloomberg.com", "state": "found", "status": 200, "robotsTxt": "https://www.bloomberg.com/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/bloomberg.com" }, { "rank": 139, "domain": "dropcatch.com", "state": "html", "status": 200, "robotsTxt": "https://www.dropcatch.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/dropcatch.com" }, { "rank": 142, "domain": "washingtonpost.com", "state": "found", "status": 200, "robotsTxt": "https://www.washingtonpost.com/robots.txt", "blocked": [ "YandexBot", "PerplexityBot", "Amazonbot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Meta-ExternalFetcher", "AhrefsBot" ], "partlyBlocked": [ "SemrushBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/washingtonpost.com" }, { "rank": 144, "domain": "stripe.com", "state": "found", "status": 200, "robotsTxt": "https://stripe.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/stripe.com" }, { "rank": 145, "domain": "calendly.com", "state": "found", "status": 200, "robotsTxt": "https://calendly.com/robots.txt", "blocked": [ "CCBot" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/calendly.com" }, { "rank": 146, "domain": "nasa.gov", "state": "found", "status": 200, "robotsTxt": "https://www.nasa.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nasa.gov" }, { "rank": 147, "domain": "wsj.com", "state": "found", "status": 200, "robotsTxt": "https://www.wsj.com/robots.txt", "blocked": [ "PetalBot", "YandexBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "Baiduspider", "OAI-SearchBot", "GPTBot", "GoogleOther", "ChatGPT-User", "AhrefsBot", "SemrushBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/wsj.com" }, { "rank": 148, "domain": "msn.com", "state": "found", "status": 200, "robotsTxt": "https://www.msn.com/robots.txt", "blocked": [ "Applebot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/msn.com" }, { "rank": 149, "domain": "arxiv.org", "state": "found", "status": 200, "robotsTxt": "https://arxiv.org/robots.txt", "blocked": [ "SemrushBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "YandexBot", "Baiduspider" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/arxiv.org" }, { "rank": 150, "domain": "unsplash.com", "state": "found", "status": 200, "robotsTxt": "https://unsplash.com/robots.txt", "blocked": [ "Claude-SearchBot", "ClaudeBot", "Bytespider", "AhrefsBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/unsplash.com" }, { "rank": 153, "domain": "trustpilot.com", "state": "found", "status": 200, "robotsTxt": "https://www.trustpilot.com/robots.txt", "blocked": [ "PetalBot", "YandexBot", "DuckAssistBot", "Amazonbot", "YouBot", "Applebot-Extended", "GoogleOther", "Bytespider", "Perplexity-User", "MistralAI-User", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "ChatGPT-User", "Claude-User", "Meta-ExternalFetcher" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/trustpilot.com" }, { "rank": 154, "domain": "statista.com", "state": "found", "status": 200, "robotsTxt": "https://www.statista.com/robots.txt", "blocked": [ "PerplexityBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "Perplexity-User", "Meta-ExternalFetcher" ], "partlyBlocked": [ "AhrefsBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/statista.com" }, { "rank": 155, "domain": "businessinsider.com", "state": "found", "status": 200, "robotsTxt": "https://www.businessinsider.com/robots.txt", "blocked": [ "ClaudeBot", "CCBot", "Bytespider" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/businessinsider.com" }, { "rank": 156, "domain": "gitlab.com", "state": "found", "status": 200, "robotsTxt": "https://gitlab.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gitlab.com" }, { "rank": 157, "domain": "ietf.org", "state": "found", "status": 200, "robotsTxt": "https://www.ietf.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ietf.org" }, { "rank": 158, "domain": "line.me", "state": "found", "status": 200, "robotsTxt": "https://line.me/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/line.me" }, { "rank": 161, "domain": "bsky.app", "state": "found", "status": 200, "robotsTxt": "https://bsky.app/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bsky.app" }, { "rank": 163, "domain": "substack.com", "state": "found", "status": 200, "robotsTxt": "https://substack.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/substack.com" }, { "rank": 164, "domain": "canva.com", "state": "found", "status": 200, "robotsTxt": "https://www.canva.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/canva.com" }, { "rank": 165, "domain": "mysql.com", "state": "found", "status": 200, "robotsTxt": "https://www.mysql.com/robots.txt", "blocked": [], "partlyBlocked": [ "Bingbot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/mysql.com" }, { "rank": 166, "domain": "twitch.tv", "state": "found", "status": 200, "robotsTxt": "https://www.twitch.tv/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/twitch.tv" }, { "rank": 168, "domain": "cloudflare.com", "state": "found", "status": 200, "robotsTxt": "https://www.cloudflare.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/cloudflare.com" }, { "rank": 169, "domain": "cnbc.com", "state": "found", "status": 200, "robotsTxt": "https://www.cnbc.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User" ], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/cnbc.com" }, { "rank": 171, "domain": "eventbrite.com", "state": "found", "status": 200, "robotsTxt": "https://eventbrite.com/robots.txt", "blocked": [ "CCBot", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/eventbrite.com" }, { "rank": 172, "domain": "oup.com", "state": "html", "status": 202, "robotsTxt": "http://global.oup.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/oup.com" }, { "rank": 174, "domain": "etsy.com", "state": "found", "status": 200, "robotsTxt": "https://www.etsy.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/etsy.com" }, { "rank": 176, "domain": "wired.com", "state": "found", "status": 200, "robotsTxt": "https://www.wired.com/robots.txt", "blocked": [ "PetalBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/wired.com" }, { "rank": 177, "domain": "openstreetmap.org", "state": "found", "status": 200, "robotsTxt": "https://www.openstreetmap.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/openstreetmap.org" }, { "rank": 179, "domain": "time.com", "state": "found", "status": 200, "robotsTxt": "https://time.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/time.com" }, { "rank": 180, "domain": "npr.org", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/npr.org" }, { "rank": 182, "domain": "samsung.com", "state": "found", "status": 200, "robotsTxt": "https://www.samsung.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot", "OAI-SearchBot", "PerplexityBot", "GPTBot", "Google-Extended", "ChatGPT-User" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/samsung.com" }, { "rank": 183, "domain": "stackoverflow.com", "state": "none", "status": 418, "robotsTxt": "https://stackoverflow.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/stackoverflow.com" }, { "rank": 184, "domain": "behance.net", "state": "found", "status": 200, "robotsTxt": "https://www.behance.net/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/behance.net" }, { "rank": 186, "domain": "slideshare.net", "state": "html", "status": 200, "robotsTxt": "https://slideshare.net/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/slideshare.net" }, { "rank": 187, "domain": "tandfonline.com", "state": "found", "status": 200, "robotsTxt": "https://www.tandfonline.com/robots.txt", "blocked": [ "GPTBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/tandfonline.com" }, { "rank": 188, "domain": "hp.com", "state": "found", "status": 200, "robotsTxt": "https://www.hp.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/hp.com" }, { "rank": 189, "domain": "ebay.com", "state": "found", "status": 200, "robotsTxt": "https://www.ebay.com/robots.txt", "blocked": [ "PerplexityBot", "Amazonbot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "OAI-SearchBot", "Claude-SearchBot", "ChatGPT-User", "Claude-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/ebay.com" }, { "rank": 191, "domain": "unesco.org", "state": "found", "status": 200, "robotsTxt": "https://www.unesco.org/robots.txt", "blocked": [ "Amazonbot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/unesco.org" }, { "rank": 192, "domain": "chatgpt.com", "state": "found", "status": 200, "robotsTxt": "https://chatgpt.com/robots.txt", "blocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/chatgpt.com" }, { "rank": 193, "domain": "techcrunch.com", "state": "found", "status": 200, "robotsTxt": "https://techcrunch.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Bytespider", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/techcrunch.com" }, { "rank": 195, "domain": "hubspot.com", "state": "found", "status": 200, "robotsTxt": "https://www.hubspot.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/hubspot.com" }, { "rank": 198, "domain": "iso.org", "state": "refused", "status": 403, "robotsTxt": "https://iso.org/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/iso.org" }, { "rank": 199, "domain": "ft.com", "state": "found", "status": 200, "robotsTxt": "https://www.ft.com/robots.txt", "blocked": [ "PerplexityBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "Perplexity-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ft.com" }, { "rank": 200, "domain": "sagepub.com", "state": "found", "status": 200, "robotsTxt": "https://www.sagepub.com/robots.txt", "blocked": [ "Amazonbot", "CCBot", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/sagepub.com" }, { "rank": 201, "domain": "mdpi.com", "state": "refused", "status": 403, "robotsTxt": "https://mdpi.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/mdpi.com" }, { "rank": 204, "domain": "loc.gov", "state": "found", "status": 200, "robotsTxt": "https://www.loc.gov/robots.txt", "blocked": [ "Baiduspider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/loc.gov" }, { "rank": 205, "domain": "go.com", "state": "found", "status": 200, "robotsTxt": "https://www.go.com/robots.txt", "blocked": [ "GPTBot", "Google-Extended" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/go.com" }, { "rank": 206, "domain": "berkeley.edu", "state": "found", "status": 200, "robotsTxt": "https://www.berkeley.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/berkeley.edu" }, { "rank": 211, "domain": "cornell.edu", "state": "found", "status": 200, "robotsTxt": "https://www.cornell.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cornell.edu" }, { "rank": 213, "domain": "ca.gov", "state": "found", "status": 200, "robotsTxt": "https://www.ca.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ca.gov" }, { "rank": 214, "domain": "usatoday.com", "state": "found", "status": 200, "robotsTxt": "https://www.usatoday.com/robots.txt", "blocked": [ "Applebot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/usatoday.com" }, { "rank": 215, "domain": "telegraph.co.uk", "state": "found", "status": 200, "robotsTxt": "https://www.telegraph.co.uk/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [ "Google-Extended" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/telegraph.co.uk" }, { "rank": 216, "domain": "booking.com", "state": "found", "status": 200, "robotsTxt": "https://booking.com/robots.txt", "blocked": [], "partlyBlocked": [ "Applebot", "Baiduspider" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/booking.com" }, { "rank": 217, "domain": "bilibili.com", "state": "found", "status": 200, "robotsTxt": "https://www.bilibili.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/bilibili.com" }, { "rank": 218, "domain": "jotform.com", "state": "found", "status": 200, "robotsTxt": "https://www.jotform.com/robots.txt", "blocked": [], "partlyBlocked": [ "Bingbot", "YandexBot", "PerplexityBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Meta-ExternalFetcher" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/jotform.com" }, { "rank": 219, "domain": "fda.gov", "state": "none", "status": 404, "robotsTxt": "https://www.fda.gov/apology_objects/abuse-detection-apology.html", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/fda.gov" }, { "rank": 220, "domain": "zendesk.com", "state": "found", "status": 200, "robotsTxt": "https://www.zendesk.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/zendesk.com" }, { "rank": 221, "domain": "tripadvisor.com", "state": "found", "status": 200, "robotsTxt": "https://www.tripadvisor.com/robots.txt", "blocked": [ "PetalBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [ "Bingbot", "Applebot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "ChatGPT-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/tripadvisor.com" }, { "rank": 222, "domain": "squarespace.com", "state": "found", "status": 200, "robotsTxt": "https://www.squarespace.com/robots.txt", "blocked": [], "partlyBlocked": [ "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/squarespace.com" }, { "rank": 223, "domain": "yandex.ru", "state": "found", "status": 200, "robotsTxt": "https://yandex.ru/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "OAI-SearchBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/yandex.ru" }, { "rank": 224, "domain": "aliyun.com", "state": "found", "status": 200, "robotsTxt": "https://www.aliyun.com/robots.txt", "blocked": [], "partlyBlocked": [ "GPTBot", "ClaudeBot", "CCBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/aliyun.com" }, { "rank": 225, "domain": "rfc-editor.org", "state": "found", "status": 200, "robotsTxt": "https://www.rfc-editor.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/rfc-editor.org" }, { "rank": 227, "domain": "cnet.com", "state": "found", "status": 200, "robotsTxt": "https://www.cnet.com/robots.txt", "blocked": [ "Applebot", "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cnet.com" }, { "rank": 229, "domain": "bandcamp.com", "state": "found", "status": 200, "robotsTxt": "https://bandcamp.com/robots.txt", "blocked": [ "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "SemrushBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bandcamp.com" }, { "rank": 232, "domain": "blogger.com", "state": "found", "status": 200, "robotsTxt": "https://www.blogger.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/blogger.com" }, { "rank": 235, "domain": "britannica.com", "state": "found", "status": 200, "robotsTxt": "https://www.britannica.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/britannica.com" }, { "rank": 236, "domain": "sina.com.cn", "state": "found", "status": 200, "robotsTxt": "https://www.sina.com.cn/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/sina.com.cn" }, { "rank": 238, "domain": "cbsnews.com", "state": "found", "status": 200, "robotsTxt": "https://www.cbsnews.com/robots.txt", "blocked": [ "GPTBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cbsnews.com" }, { "rank": 239, "domain": "surveymonkey.com", "state": "found", "status": 200, "robotsTxt": "https://www.surveymonkey.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/surveymonkey.com" }, { "rank": 240, "domain": "mckinsey.com", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/mckinsey.com" }, { "rank": 241, "domain": "ted.com", "state": "found", "status": 200, "robotsTxt": "https://www.ted.com/robots.txt", "blocked": [ "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [ "PetalBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "YouBot", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/ted.com" }, { "rank": 243, "domain": "free.fr", "state": "found", "status": 200, "robotsTxt": "https://www.free.fr/robots.txt", "blocked": [ "Baiduspider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/free.fr" }, { "rank": 244, "domain": "pixabay.com", "state": "found", "status": 200, "robotsTxt": "https://pixabay.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Bytespider", "AhrefsBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/pixabay.com" }, { "rank": 245, "domain": "redhat.com", "state": "found", "status": 200, "robotsTxt": "https://www.redhat.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/redhat.com" }, { "rank": 246, "domain": "sedo.com", "state": "refused", "status": 503, "robotsTxt": "https://sedo.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/sedo.com" }, { "rank": 247, "domain": "cambridge.org", "state": "found", "status": 200, "robotsTxt": "https://www.cambridge.org/robots.txt", "blocked": [ "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cambridge.org" }, { "rank": 249, "domain": "wix.com", "state": "found", "status": 200, "robotsTxt": "https://www.wix.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/wix.com" }, { "rank": 250, "domain": "prnewswire.com", "state": "found", "status": 200, "robotsTxt": "https://www.prnewswire.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/prnewswire.com" }, { "rank": 251, "domain": "hugedomains.com", "state": "found", "status": 200, "robotsTxt": "https://www.hugedomains.com/robots.txt", "blocked": [ "DotBot" ], "partlyBlocked": [ "Googlebot", "Applebot", "PerplexityBot", "GPTBot", "ClaudeBot", "Google-Extended" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/hugedomains.com" }, { "rank": 253, "domain": "ieee.org", "state": "html", "status": 202, "robotsTxt": "https://www.ieee.org/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/ieee.org" }, { "rank": 254, "domain": "independent.co.uk", "state": "found", "status": 200, "robotsTxt": "https://www.independent.co.uk/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/independent.co.uk" }, { "rank": 256, "domain": "mailchimp.com", "state": "found", "status": 200, "robotsTxt": "https://mailchimp.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/mailchimp.com" }, { "rank": 257, "domain": "163.com", "state": "found", "status": 200, "robotsTxt": "http://www.163.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/163.com" }, { "rank": 258, "domain": "nist.gov", "state": "found", "status": 200, "robotsTxt": "https://www.nist.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nist.gov" }, { "rank": 259, "domain": "dailymotion.com", "state": "found", "status": 200, "robotsTxt": "https://www.dailymotion.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot", "YandexBot" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/dailymotion.com" }, { "rank": 260, "domain": "theverge.com", "state": "found", "status": 200, "robotsTxt": "https://theverge.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/theverge.com" }, { "rank": 261, "domain": "hbr.org", "state": "found", "status": 200, "robotsTxt": "https://hbr.org/robots.txt", "blocked": [ "GPTBot", "Google-Extended", "CCBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/hbr.org" }, { "rank": 262, "domain": "frontiersin.org", "state": "found", "status": 200, "robotsTxt": "https://www.frontiersin.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/frontiersin.org" }, { "rank": 265, "domain": "steampowered.com", "state": "found", "status": 200, "robotsTxt": "https://store.steampowered.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/steampowered.com" }, { "rank": 266, "domain": "princeton.edu", "state": "found", "status": 200, "robotsTxt": "https://www.princeton.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/princeton.edu" }, { "rank": 267, "domain": "docker.com", "state": "found", "status": 200, "robotsTxt": "https://www.docker.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/docker.com" }, { "rank": 268, "domain": "worldbank.org", "state": "found", "status": 200, "robotsTxt": "https://www.worldbank.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/worldbank.org" }, { "rank": 269, "domain": "yelp.com", "state": "found", "status": 200, "robotsTxt": "https://www.yelp.com/robots.txt", "blocked": [ "DuckDuckBot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/yelp.com" }, { "rank": 270, "domain": "huawei.com", "state": "found", "status": 200, "robotsTxt": "https://www.huawei.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/huawei.com" }, { "rank": 271, "domain": "epa.gov", "state": "found", "status": 200, "robotsTxt": "https://www.epa.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/epa.gov" }, { "rank": 274, "domain": "taobao.com", "state": "found", "status": 200, "robotsTxt": "https://www.taobao.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/taobao.com" }, { "rank": 275, "domain": "usda.gov", "state": "refused", "status": 403, "robotsTxt": "https://www.usda.gov/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/usda.gov" }, { "rank": 276, "domain": "python.org", "state": "found", "status": 200, "robotsTxt": "https://www.python.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/python.org" }, { "rank": 277, "domain": "myspace.com", "state": "found", "status": 200, "robotsTxt": "https://myspace.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/myspace.com" }, { "rank": 278, "domain": "telegram.org", "state": "none", "status": 404, "robotsTxt": "https://telegram.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/telegram.org" }, { "rank": 281, "domain": "netflix.com", "state": "found", "status": 200, "robotsTxt": "https://www.netflix.com/robots.txt", "blocked": [ "DuckDuckBot", "PetalBot", "DuckAssistBot", "Amazonbot", "YouBot", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "GPTBot", "ClaudeBot", "Google-Extended", "ChatGPT-User", "Claude-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/netflix.com" }, { "rank": 282, "domain": "imgur.com", "state": "found", "status": 200, "robotsTxt": "https://imgur.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/imgur.com" }, { "rank": 283, "domain": "latimes.com", "state": "found", "status": 200, "robotsTxt": "https://www.latimes.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/latimes.com" }, { "rank": 284, "domain": "goodreads.com", "state": "found", "status": 200, "robotsTxt": "https://www.goodreads.com/robots.txt", "blocked": [ "Bingbot", "GPTBot", "CCBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/goodreads.com" }, { "rank": 285, "domain": "ubuntu.com", "state": "found", "status": 200, "robotsTxt": "https://ubuntu.com/robots.txt", "blocked": [], "partlyBlocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "GPTBot", "ClaudeBot", "Google-Extended", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Perplexity-User" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/ubuntu.com" }, { "rank": 286, "domain": "oecd.org", "state": "found", "status": 200, "robotsTxt": "https://www.oecd.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/oecd.org" }, { "rank": 287, "domain": "nvidia.com", "state": "found", "status": 200, "robotsTxt": "https://www.nvidia.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/nvidia.com" }, { "rank": 288, "domain": "weforum.org", "state": "refused", "status": 403, "robotsTxt": "https://www.weforum.org/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/weforum.org" }, { "rank": 290, "domain": "mps.gov.cn", "state": "error", "status": 521, "robotsTxt": "https://www.mps.gov.cn/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/mps.gov.cn" }, { "rank": 291, "domain": "whitehouse.gov", "state": "found", "status": 200, "robotsTxt": "https://www.whitehouse.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/whitehouse.gov" }, { "rank": 292, "domain": "washington.edu", "state": "found", "status": 200, "robotsTxt": "https://www.washington.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/washington.edu" }, { "rank": 293, "domain": "nationalgeographic.com", "state": "found", "status": 200, "robotsTxt": "https://www.nationalgeographic.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nationalgeographic.com" }, { "rank": 294, "domain": "apnews.com", "state": "found", "status": 200, "robotsTxt": "https://apnews.com/robots.txt", "blocked": [ "PerplexityBot", "Claude-SearchBot", "Amazonbot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Claude-User", "Perplexity-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/apnews.com" }, { "rank": 295, "domain": "investopedia.com", "state": "found", "status": 200, "robotsTxt": "https://www.investopedia.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [ "OAI-SearchBot", "GPTBot", "ChatGPT-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/investopedia.com" }, { "rank": 298, "domain": "deloitte.com", "state": "found", "status": 200, "robotsTxt": "https://www.deloitte.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/deloitte.com" }, { "rank": 299, "domain": "scribd.com", "state": "html", "status": 200, "robotsTxt": "https://www.scribd.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/scribd.com" }, { "rank": 300, "domain": "academia.edu", "state": "found", "status": 200, "robotsTxt": "https://www.academia.edu/robots.txt", "blocked": [ "PetalBot", "YandexBot", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "Perplexity-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "YouBot", "ChatGPT-User", "Claude-User", "MistralAI-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/academia.edu" }, { "rank": 301, "domain": "youku.com", "state": "html", "status": 200, "robotsTxt": "https://www.youku.com/index/y404", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/youku.com" }, { "rank": 303, "domain": "safety.google", "state": "found", "status": 200, "robotsTxt": "https://safety.google/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/safety.google" }, { "rank": 304, "domain": "ovhcloud.com", "state": "found", "status": 200, "robotsTxt": "https://www.ovhcloud.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ovhcloud.com" }, { "rank": 305, "domain": "pexels.com", "state": "found", "status": 200, "robotsTxt": "https://www.pexels.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/pexels.com" }, { "rank": 306, "domain": "themeforest.net", "state": "found", "status": 200, "robotsTxt": "https://themeforest.net/robots.txt", "blocked": [ "CCBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/themeforest.net" }, { "rank": 307, "domain": "fandom.com", "state": "refused", "status": 403, "robotsTxt": "https://www.fandom.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/fandom.com" }, { "rank": 308, "domain": "indiatimes.com", "state": "found", "status": 200, "robotsTxt": "https://www.indiatimes.com/robots.txt", "blocked": [ "Meta-ExternalAgent" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/indiatimes.com" }, { "rank": 309, "domain": "noaa.gov", "state": "found", "status": 200, "robotsTxt": "https://www.noaa.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/noaa.gov" }, { "rank": 310, "domain": "mayoclinic.org", "state": "refused", "status": 403, "robotsTxt": "https://www.mayoclinic.org/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/mayoclinic.org" }, { "rank": 311, "domain": "buydomains.com", "state": "refused", "status": 403, "robotsTxt": "https://buydomains.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/buydomains.com" }, { "rank": 312, "domain": "ox.ac.uk", "state": "refused", "status": 403, "robotsTxt": "https://www.ox.ac.uk/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/ox.ac.uk" }, { "rank": 313, "domain": "theatlantic.com", "state": "found", "status": 200, "robotsTxt": "https://www.theatlantic.com/robots.txt", "blocked": [ "PetalBot", "PerplexityBot", "Claude-SearchBot", "Amazonbot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "OAI-SearchBot", "DuckAssistBot", "GPTBot", "ChatGPT-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/theatlantic.com" }, { "rank": 314, "domain": "state.gov", "state": "refused", "status": 403, "robotsTxt": "https://www.state.gov/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/state.gov" }, { "rank": 315, "domain": "marriott.com", "state": "found", "status": 200, "robotsTxt": "https://www.marriott.com/robots.txt", "blocked": [ "Baiduspider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/marriott.com" }, { "rank": 316, "domain": "plos.org", "state": "found", "status": 200, "robotsTxt": "https://plos.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/plos.org" }, { "rank": 318, "domain": "nbcnews.com", "state": "found", "status": 200, "robotsTxt": "https://www.nbcnews.com/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nbcnews.com" }, { "rank": 319, "domain": "cisco.com", "state": "found", "status": 200, "robotsTxt": "https://www.cisco.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/cisco.com" }, { "rank": 320, "domain": "tencent.com", "state": "found", "status": 200, "robotsTxt": "https://www.tencent.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/tencent.com" }, { "rank": 324, "domain": "huffpost.com", "state": "found", "status": 200, "robotsTxt": "https://www.huffpost.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "Meta-ExternalFetcher" ], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/huffpost.com" }, { "rank": 325, "domain": "columbia.edu", "state": "none", "status": 404, "robotsTxt": "https://columbia.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/columbia.edu" }, { "rank": 326, "domain": "aol.com", "state": "found", "status": 200, "robotsTxt": "https://www.aol.com/robots.txt", "blocked": [ "PetalBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Bytespider" ], "partlyBlocked": [ "PerplexityBot", "ChatGPT-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/aol.com" }, { "rank": 327, "domain": "sohu.com", "state": "found", "status": 200, "robotsTxt": "https://www.sohu.com/robots.txt", "blocked": [ "DuckDuckBot", "Applebot", "PetalBot", "YandexBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Baiduspider", "Bytespider" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/sohu.com" }, { "rank": 329, "domain": "apa.org", "state": "found", "status": 200, "robotsTxt": "https://www.apa.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/apa.org" }, { "rank": 330, "domain": "livejournal.com", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/livejournal.com" }, { "rank": 331, "domain": "xing.com", "state": "found", "status": 200, "robotsTxt": "https://www.xing.com/robots.txt", "blocked": [], "partlyBlocked": [ "PerplexityBot", "Claude-SearchBot", "GPTBot", "ClaudeBot", "Claude-User", "Perplexity-User" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/xing.com" }, { "rank": 332, "domain": "arcgis.com", "state": "found", "status": 200, "robotsTxt": "https://www.arcgis.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/arcgis.com" }, { "rank": 333, "domain": "intel.com", "state": "found", "status": 200, "robotsTxt": "https://www.intel.com/robots.txt", "blocked": [ "AhrefsBot", "SemrushBot" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/intel.com" }, { "rank": 337, "domain": "indeed.com", "state": "found", "status": 200, "robotsTxt": "https://www.indeed.com/robots.txt", "blocked": [ "GoogleOther" ], "partlyBlocked": [ "Googlebot", "DuckDuckBot", "Applebot", "PetalBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "Meta-ExternalFetcher" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/indeed.com" }, { "rank": 338, "domain": "odoo.com", "state": "found", "status": 200, "robotsTxt": "https://www.odoo.com/robots.txt", "blocked": [ "PetalBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/odoo.com" }, { "rank": 339, "domain": "webmd.com", "state": "found", "status": 200, "robotsTxt": "https://www.webmd.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "CCBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/webmd.com" }, { "rank": 340, "domain": "patreon.com", "state": "found", "status": 200, "robotsTxt": "https://patreon.com/robots.txt", "blocked": [ "PerplexityBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/patreon.com" }, { "rank": 341, "domain": "yale.edu", "state": "found", "status": 200, "robotsTxt": "https://www.yale.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/yale.edu" }, { "rank": 342, "domain": "salesforce.com", "state": "found", "status": 200, "robotsTxt": "https://www.salesforce.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/salesforce.com" }, { "rank": 345, "domain": "theconversation.com", "state": "found", "status": 200, "robotsTxt": "https://theconversation.com/robots.txt", "blocked": [ "Applebot", "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [ "Bingbot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/theconversation.com" }, { "rank": 346, "domain": "umich.edu", "state": "found", "status": 200, "robotsTxt": "https://umich.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/umich.edu" }, { "rank": 347, "domain": "kickstarter.com", "state": "refused", "status": 403, "robotsTxt": "https://www.kickstarter.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/kickstarter.com" }, { "rank": 348, "domain": "ftc.gov", "state": "found", "status": 200, "robotsTxt": "https://www.ftc.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ftc.gov" }, { "rank": 349, "domain": "pbs.org", "state": "found", "status": 200, "robotsTxt": "https://www.pbs.org/robots.txt", "blocked": [ "PerplexityBot", "GPTBot", "CCBot", "Bytespider" ], "partlyBlocked": [ "Baiduspider" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/pbs.org" }, { "rank": 351, "domain": "quora.com", "state": "found", "status": 200, "robotsTxt": "https://www.quora.com/robots.txt", "blocked": [ "Bingbot", "DuckDuckBot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Applebot", "Amazonbot", "Google-Extended" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/quora.com" }, { "rank": 352, "domain": "biomedcentral.com", "state": "none", "status": 404, "robotsTxt": "https://www.biomedcentral.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/biomedcentral.com" }, { "rank": 353, "domain": "ico.org.uk", "state": "refused", "status": 403, "robotsTxt": "https://ico.org.uk/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/ico.org.uk" }, { "rank": 354, "domain": "economist.com", "state": "found", "status": 200, "robotsTxt": "https://www.economist.com/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/economist.com" }, { "rank": 355, "domain": "dribbble.com", "state": "found", "status": 200, "robotsTxt": "https://dribbble.com/robots.txt", "blocked": [], "partlyBlocked": [ "PetalBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/dribbble.com" }, { "rank": 356, "domain": "healthline.com", "state": "found", "status": 200, "robotsTxt": "https://www.healthline.com/robots.txt", "blocked": [ "Amazonbot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/healthline.com" }, { "rank": 357, "domain": "change.org", "state": "found", "status": 200, "robotsTxt": "https://www.change.org/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "YandexBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/change.org" }, { "rank": 359, "domain": "canada.ca", "state": "found", "status": 200, "robotsTxt": "https://www.canada.ca/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/canada.ca" }, { "rank": 360, "domain": "android.com", "state": "found", "status": 200, "robotsTxt": "https://www.android.com/robots.txt", "blocked": [], "partlyBlocked": [ "Applebot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "MistralAI-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/android.com" }, { "rank": 362, "domain": "businesswire.com", "state": "refused", "status": 403, "robotsTxt": "https://businesswire.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/businesswire.com" }, { "rank": 366, "domain": "postgresql.org", "state": "found", "status": 200, "robotsTxt": "https://www.postgresql.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/postgresql.org" }, { "rank": 367, "domain": "eff.org", "state": "found", "status": 200, "robotsTxt": "https://www.eff.org/robots.txt", "blocked": [], "partlyBlocked": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "AhrefsBot", "SemrushBot" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/eff.org" }, { "rank": 368, "domain": "alibaba.com", "state": "found", "status": 200, "robotsTxt": "https://www.alibaba.com/atlassitemapsitenet/static/www_alibaba_com/robots.txt", "blocked": [ "GPTBot", "Google-Extended" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/alibaba.com" }, { "rank": 369, "domain": "bmj.com", "state": "found", "status": 200, "robotsTxt": "https://www.bmj.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bmj.com" }, { "rank": 373, "domain": "bbb.org", "state": "found", "status": 200, "robotsTxt": "https://www.bbb.org/robots.txt", "blocked": [], "partlyBlocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "ChatGPT-User", "Claude-User", "Perplexity-User" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/bbb.org" }, { "rank": 374, "domain": "fortune.com", "state": "found", "status": 200, "robotsTxt": "https://fortune.com/robots.txt", "blocked": [ "CCBot" ], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/fortune.com" }, { "rank": 376, "domain": "upenn.edu", "state": "found", "status": 200, "robotsTxt": "https://upenn.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/upenn.edu" }, { "rank": 377, "domain": "typeform.com", "state": "found", "status": 200, "robotsTxt": "https://www.typeform.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/typeform.com" }, { "rank": 379, "domain": "psu.edu", "state": "found", "status": 200, "robotsTxt": "https://www.psu.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/psu.edu" }, { "rank": 380, "domain": "admin.ch", "state": "found", "status": 200, "robotsTxt": "https://www.admin.ch/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/admin.ch" }, { "rank": 381, "domain": "ovh.com", "state": "found", "status": 200, "robotsTxt": "https://www.ovh.com/robots.txt", "blocked": [ "GPTBot", "CCBot", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ovh.com" }, { "rank": 383, "domain": "gartner.com", "state": "found", "status": 200, "robotsTxt": "https://www.gartner.com/robots.txt", "blocked": [ "OAI-SearchBot", "GPTBot", "Google-Extended", "CCBot", "ChatGPT-User" ], "partlyBlocked": [ "ClaudeBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/gartner.com" }, { "rank": 384, "domain": "jimdo.com", "state": "found", "status": 200, "robotsTxt": "https://www.jimdo.com/robots.txt", "blocked": [], "partlyBlocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/jimdo.com" }, { "rank": 386, "domain": "pewresearch.org", "state": "found", "status": 200, "robotsTxt": "https://www.pewresearch.org/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/pewresearch.org" }, { "rank": 387, "domain": "cbc.ca", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/cbc.ca" }, { "rank": 388, "domain": "psychologytoday.com", "state": "found", "status": 200, "robotsTxt": "https://www.psychologytoday.com/robots.txt", "blocked": [ "SemrushBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/psychologytoday.com" }, { "rank": 389, "domain": "foxnews.com", "state": "found", "status": 200, "robotsTxt": "https://www.foxnews.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/foxnews.com" }, { "rank": 390, "domain": "fao.org", "state": "found", "status": 200, "robotsTxt": "https://www.fao.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/fao.org" }, { "rank": 391, "domain": "zhihu.com", "state": "found", "status": 200, "robotsTxt": "https://www.zhihu.com/robots.txt", "blocked": [ "DuckDuckBot", "PetalBot", "YandexBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "Baiduspider" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/zhihu.com" }, { "rank": 392, "domain": "deviantart.com", "state": "found", "status": 200, "robotsTxt": "https://www.deviantart.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/deviantart.com" }, { "rank": 393, "domain": "wpastra.com", "state": "found", "status": 200, "robotsTxt": "https://wpastra.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/wpastra.com" }, { "rank": 394, "domain": "bund.de", "state": "found", "status": 200, "robotsTxt": "https://verwaltung.bund.de/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bund.de" }, { "rank": 395, "domain": "cam.ac.uk", "state": "found", "status": 200, "robotsTxt": "https://www.cam.ac.uk/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cam.ac.uk" }, { "rank": 396, "domain": "jstor.org", "state": "found", "status": 200, "robotsTxt": "https://www.jstor.org/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot" ], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/jstor.org" }, { "rank": 397, "domain": "moodle.org", "state": "found", "status": 200, "robotsTxt": "https://moodle.org/robots.txt", "blocked": [], "partlyBlocked": [ "AhrefsBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/moodle.org" }, { "rank": 398, "domain": "firefox.com", "state": "found", "status": 200, "robotsTxt": "https://www.firefox.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/firefox.com" }, { "rank": 399, "domain": "usnews.com", "state": "found", "status": 200, "robotsTxt": "https://www.usnews.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/usnews.com" }, { "rank": 401, "domain": "letsencrypt.org", "state": "found", "status": 200, "robotsTxt": "https://letsencrypt.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/letsencrypt.org" }, { "rank": 402, "domain": "uci.edu", "state": "none", "status": 404, "robotsTxt": "https://uci.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/uci.edu" }, { "rank": 404, "domain": "zdnet.com", "state": "found", "status": 200, "robotsTxt": "https://www.zdnet.com/robots.txt", "blocked": [ "Bingbot", "Applebot", "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/zdnet.com" }, { "rank": 405, "domain": "akismet.com", "state": "found", "status": 200, "robotsTxt": "https://akismet.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/akismet.com" }, { "rank": 407, "domain": "abc.net.au", "state": "found", "status": 200, "robotsTxt": "https://www.abc.net.au/robots.txt", "blocked": [ "PerplexityBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/abc.net.au" }, { "rank": 408, "domain": "gofundme.com", "state": "found", "status": 200, "robotsTxt": "https://www.gofundme.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gofundme.com" }, { "rank": 409, "domain": "sciencedaily.com", "state": "found", "status": 200, "robotsTxt": "https://sciencedaily.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/sciencedaily.com" }, { "rank": 410, "domain": "coursera.org", "state": "found", "status": 200, "robotsTxt": "https://www.coursera.org/robots.txt", "blocked": [ "Meta-ExternalAgent" ], "partlyBlocked": [ "OAI-SearchBot", "PerplexityBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "ChatGPT-User", "MistralAI-User" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/coursera.org" }, { "rank": 411, "domain": "nypost.com", "state": "found", "status": 200, "robotsTxt": "https://nypost.com/robots.txt", "blocked": [ "PetalBot", "PerplexityBot", "Claude-SearchBot", "Amazonbot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Meta-ExternalFetcher", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nypost.com" }, { "rank": 413, "domain": "umn.edu", "state": "found", "status": 200, "robotsTxt": "https://www.umn.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/umn.edu" }, { "rank": 414, "domain": "elsevier.com", "state": "found", "status": 200, "robotsTxt": "https://www.elsevier.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/elsevier.com" }, { "rank": 416, "domain": "dailymail.com", "state": "found", "status": 200, "robotsTxt": "https://www.dailymail.com/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/dailymail.com" }, { "rank": 417, "domain": "acm.org", "state": "found", "status": 200, "robotsTxt": "https://www.acm.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/acm.org" }, { "rank": 418, "domain": "xinhuanet.com", "state": "found", "status": 200, "robotsTxt": "https://www.xinhuanet.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/xinhuanet.com" }, { "rank": 419, "domain": "elegantthemes.com", "state": "found", "status": 200, "robotsTxt": "https://www.elegantthemes.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/elegantthemes.com" }, { "rank": 420, "domain": "newyorker.com", "state": "found", "status": 200, "robotsTxt": "https://www.newyorker.com/robots.txt", "blocked": [ "PetalBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/newyorker.com" }, { "rank": 423, "domain": "vice.com", "state": "found", "status": 200, "robotsTxt": "https://www.vice.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/vice.com" }, { "rank": 425, "domain": "newsweek.com", "state": "found", "status": 200, "robotsTxt": "https://www.newsweek.com/robots.txt", "blocked": [ "PerplexityBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Bytespider", "ChatGPT-User", "Perplexity-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/newsweek.com" }, { "rank": 426, "domain": "thelancet.com", "state": "found", "status": 200, "robotsTxt": "https://www.thelancet.com/robots.txt", "blocked": [ "PerplexityBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "MistralAI-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/thelancet.com" }, { "rank": 428, "domain": "scientificamerican.com", "state": "found", "status": 200, "robotsTxt": "https://www.scientificamerican.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "ChatGPT-User", "Claude-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/scientificamerican.com" }, { "rank": 429, "domain": "kernel.org", "state": "none", "status": 404, "robotsTxt": "https://kernel.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/kernel.org" }, { "rank": 430, "domain": "arstechnica.com", "state": "found", "status": 200, "robotsTxt": "https://arstechnica.com/robots.txt", "blocked": [ "PetalBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/arstechnica.com" }, { "rank": 431, "domain": "oreilly.com", "state": "found", "status": 200, "robotsTxt": "https://www.oreilly.com/robots.txt", "blocked": [ "PetalBot", "YandexBot", "Baiduspider", "Amazonbot", "GPTBot", "CCBot", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "YouBot", "ClaudeBot", "Google-Extended", "Applebot-Extended", "ChatGPT-User", "Perplexity-User", "MistralAI-User", "AhrefsBot", "DotBot" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/oreilly.com" }, { "rank": 432, "domain": "irs.gov", "state": "found", "status": 200, "robotsTxt": "https://www.irs.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/irs.gov" }, { "rank": 433, "domain": "dw.com", "state": "found", "status": 200, "robotsTxt": "https://www.dw.com/robots.txt", "blocked": [ "Applebot", "PetalBot", "OAI-SearchBot", "PerplexityBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Perplexity-User", "Meta-ExternalFetcher", "AhrefsBot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/dw.com" }, { "rank": 434, "domain": "threads.com", "state": "found", "status": 200, "robotsTxt": "https://www.threads.com/robots.txt", "blocked": [ "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/threads.com" }, { "rank": 435, "domain": "orcid.org", "state": "found", "status": 200, "robotsTxt": "https://orcid.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/orcid.org" }, { "rank": 436, "domain": "themeisle.com", "state": "found", "status": 200, "robotsTxt": "https://themeisle.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/themeisle.com" }, { "rank": 437, "domain": "nps.gov", "state": "found", "status": 200, "robotsTxt": "https://www.nps.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nps.gov" }, { "rank": 439, "domain": "uchicago.edu", "state": "found", "status": 200, "robotsTxt": "https://www.uchicago.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/uchicago.edu" }, { "rank": 441, "domain": "jamanetwork.com", "state": "found", "status": 200, "robotsTxt": "https://jamanetwork.com/robots.txt", "blocked": [ "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/jamanetwork.com" }, { "rank": 442, "domain": "dell.com", "state": "found", "status": 200, "robotsTxt": "https://www.dell.com/robots.txt", "blocked": [ "Baiduspider" ], "partlyBlocked": [ "Bytespider" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/dell.com" }, { "rank": 444, "domain": "git-scm.com", "state": "none", "status": 404, "robotsTxt": "https://git-scm.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/git-scm.com" }, { "rank": 447, "domain": "stackexchange.com", "state": "none", "status": 418, "robotsTxt": "https://stackexchange.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/stackexchange.com" }, { "rank": 449, "domain": "mitre.org", "state": "refused", "status": 403, "robotsTxt": "https://www.mitre.org/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/mitre.org" }, { "rank": 450, "domain": "naver.com", "state": "found", "status": 200, "robotsTxt": "https://www.naver.com/robots.txt", "blocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/naver.com" }, { "rank": 451, "domain": "cmu.edu", "state": "found", "status": 200, "robotsTxt": "https://www.cmu.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cmu.edu" }, { "rank": 452, "domain": "blog.google", "state": "found", "status": 200, "robotsTxt": "https://blog.google/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/blog.google" }, { "rank": 453, "domain": "fastcompany.com", "state": "found", "status": 200, "robotsTxt": "https://www.fastcompany.com/robots.txt", "blocked": [ "PerplexityBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Bytespider", "MistralAI-User" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "Baiduspider", "OAI-SearchBot", "Claude-SearchBot", "Google-Extended", "Meta-ExternalAgent", "ChatGPT-User", "Claude-User", "Perplexity-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/fastcompany.com" }, { "rank": 454, "domain": "wisc.edu", "state": "found", "status": 200, "robotsTxt": "https://www.wisc.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/wisc.edu" }, { "rank": 455, "domain": "slack.com", "state": "found", "status": 200, "robotsTxt": "https://slack.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/slack.com" }, { "rank": 458, "domain": "ucla.edu", "state": "found", "status": 200, "robotsTxt": "https://www.ucla.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ucla.edu" }, { "rank": 459, "domain": "acs.org", "state": "html", "status": 200, "robotsTxt": "https://www.acs.org/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/acs.org" }, { "rank": 460, "domain": "pnas.org", "state": "found", "status": 200, "robotsTxt": "https://www.pnas.org/robots.txt", "blocked": [ "GPTBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/pnas.org" }, { "rank": 461, "domain": "anthropic.com", "state": "found", "status": 200, "robotsTxt": "https://www.anthropic.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/anthropic.com" }, { "rank": 463, "domain": "hilton.com", "state": "found", "status": 200, "robotsTxt": "https://www.hilton.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/hilton.com" }, { "rank": 464, "domain": "wiktionary.org", "state": "found", "status": 200, "robotsTxt": "https://en.wiktionary.org/robots.txt", "blocked": [], "partlyBlocked": [ "SemrushBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/wiktionary.org" }, { "rank": 466, "domain": "jd.com", "state": "found", "status": 200, "robotsTxt": "https://jd.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/jd.com" }, { "rank": 467, "domain": "gizmodo.com", "state": "found", "status": 200, "robotsTxt": "https://gizmodo.com/robots.txt", "blocked": [ "CCBot", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gizmodo.com" }, { "rank": 469, "domain": "bls.gov", "state": "found", "status": 200, "robotsTxt": "https://www.bls.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bls.gov" }, { "rank": 470, "domain": "getbootstrap.com", "state": "found", "status": 200, "robotsTxt": "https://getbootstrap.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/getbootstrap.com" }, { "rank": 471, "domain": "wikihow.com", "state": "found", "status": 200, "robotsTxt": "https://www.wikihow.com/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/wikihow.com" }, { "rank": 472, "domain": "si.edu", "state": "found", "status": 200, "robotsTxt": "https://www.si.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/si.edu" }, { "rank": 473, "domain": "globenewswire.com", "state": "refused", "status": 403, "robotsTxt": "https://www.globenewswire.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/globenewswire.com" }, { "rank": 474, "domain": "jhu.edu", "state": "refused", "status": 200, "robotsTxt": "https://www.jhu.edu/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/jhu.edu" }, { "rank": 475, "domain": "people.com.cn", "state": "found", "status": 200, "robotsTxt": "https://www.people.com.cn/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/people.com.cn" }, { "rank": 476, "domain": "techtarget.com", "state": "found", "status": 200, "robotsTxt": "https://www.techtarget.com/robots.txt", "blocked": [ "PetalBot" ], "partlyBlocked": [ "Amazonbot", "AhrefsBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/techtarget.com" }, { "rank": 477, "domain": "merriam-webster.com", "state": "found", "status": 200, "robotsTxt": "https://www.merriam-webster.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/merriam-webster.com" }, { "rank": 478, "domain": "usgs.gov", "state": "found", "status": 200, "robotsTxt": "https://www.usgs.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/usgs.gov" }, { "rank": 480, "domain": "elpais.com", "state": "found", "status": 200, "robotsTxt": "https://elpais.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "ClaudeBot", "CCBot", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/elpais.com" }, { "rank": 481, "domain": "huggingface.co", "state": "found", "status": 200, "robotsTxt": "https://huggingface.co/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/huggingface.co" }, { "rank": 482, "domain": "science.org", "state": "found", "status": 200, "robotsTxt": "https://www.science.org/robots.txt", "blocked": [ "GPTBot", "Google-Extended", "CCBot", "Applebot-Extended", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/science.org" }, { "rank": 483, "domain": "aljazeera.com", "state": "found", "status": 200, "robotsTxt": "https://www.aljazeera.com/robots.txt", "blocked": [ "PerplexityBot", "GPTBot", "ClaudeBot", "Bytespider", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/aljazeera.com" }, { "rank": 484, "domain": "openssl.org", "state": "found", "status": 200, "robotsTxt": "https://openssl.org/robots.txt", "blocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/openssl.org" }, { "rank": 485, "domain": "unicef.org", "state": "found", "status": 200, "robotsTxt": "https://www.unicef.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/unicef.org" }, { "rank": 486, "domain": "itu.int", "state": "found", "status": 200, "robotsTxt": "https://www.itu.int/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/itu.int" }, { "rank": 487, "domain": "euronews.com", "state": "found", "status": 200, "robotsTxt": "https://www.euronews.com/robots.txt", "blocked": [ "GPTBot", "CCBot" ], "partlyBlocked": [ "OAI-SearchBot", "PerplexityBot", "YouBot", "ChatGPT-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/euronews.com" }, { "rank": 488, "domain": "www.nhs.uk", "state": "found", "status": 200, "robotsTxt": "https://www.nhs.uk/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot", "AhrefsBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/www.nhs.uk" }, { "rank": 489, "domain": "utexas.edu", "state": "found", "status": 200, "robotsTxt": "https://www.utexas.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/utexas.edu" }, { "rank": 490, "domain": "magnific.com", "state": "found", "status": 200, "robotsTxt": "https://www.magnific.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/magnific.com" }, { "rank": 491, "domain": "intuit.com", "state": "found", "status": 200, "robotsTxt": "https://www.intuit.com/robots.txt", "blocked": [ "Baiduspider" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/intuit.com" }, { "rank": 492, "domain": "walmart.com", "state": "found", "status": 200, "robotsTxt": "https://www.walmart.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/walmart.com" }, { "rank": 493, "domain": "ssrn.com", "state": "refused", "status": 403, "robotsTxt": "https://www.ssrn.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/ssrn.com" }, { "rank": 494, "domain": "ed.gov", "state": "refused", "status": 403, "robotsTxt": "https://ed.gov/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/ed.gov" }, { "rank": 495, "domain": "iana.org", "state": "found", "status": 200, "robotsTxt": "https://www.iana.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/iana.org" }, { "rank": 496, "domain": "giphy.com", "state": "found", "status": 200, "robotsTxt": "https://giphy.com/robots.txt", "blocked": [ "PetalBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/giphy.com" }, { "rank": 499, "domain": "atlassian.com", "state": "found", "status": 200, "robotsTxt": "https://www.atlassian.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/atlassian.com" }, { "rank": 500, "domain": "sec.gov", "state": "found", "status": 200, "robotsTxt": "https://www.sec.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/sec.gov" }, { "rank": 503, "domain": "hhs.gov", "state": "refused", "status": 403, "robotsTxt": "https://www.hhs.gov/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/hhs.gov" }, { "rank": 504, "domain": "mashable.com", "state": "found", "status": 200, "robotsTxt": "https://mashable.com/robots.txt", "blocked": [ "Applebot", "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/mashable.com" }, { "rank": 506, "domain": "engadget.com", "state": "found", "status": 200, "robotsTxt": "https://engadget.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/engadget.com" }, { "rank": 507, "domain": "aliexpress.com", "state": "found", "status": 200, "robotsTxt": "https://www.aliexpress.com/robots.txt", "blocked": [ "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Bytespider", "SemrushBot" ], "partlyBlocked": [ "YandexBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/aliexpress.com" }, { "rank": 508, "domain": "w3schools.com", "state": "found", "status": 200, "robotsTxt": "https://www.w3schools.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/w3schools.com" }, { "rank": 509, "domain": "meetup.com", "state": "found", "status": 200, "robotsTxt": "https://www.meetup.com/robots.txt", "blocked": [ "GPTBot" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/meetup.com" }, { "rank": 510, "domain": "lemonde.fr", "state": "found", "status": 200, "robotsTxt": "https://www.lemonde.fr/robots.txt", "blocked": [ "PetalBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Bytespider", "Claude-User", "MistralAI-User", "SemrushBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/lemonde.fr" }, { "rank": 511, "domain": "matterport.com", "state": "found", "status": 200, "robotsTxt": "https://matterport.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/matterport.com" }, { "rank": 513, "domain": "ikea.com", "state": "found", "status": 200, "robotsTxt": "https://www.ikea.com/robots.txt", "blocked": [ "Amazonbot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ikea.com" }, { "rank": 517, "domain": "justice.gov", "state": "found", "status": 200, "robotsTxt": "https://justice.gov/robots.txt", "blocked": [ "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/justice.gov" }, { "rank": 518, "domain": "www.gov.cn", "state": "found", "status": 200, "robotsTxt": "https://www.gov.cn/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/www.gov.cn" }, { "rank": 519, "domain": "automattic.com", "state": "found", "status": 200, "robotsTxt": "https://automattic.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/automattic.com" }, { "rank": 520, "domain": "teamviewer.com", "state": "found", "status": 200, "robotsTxt": "https://www.teamviewer.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/teamviewer.com" }, { "rank": 521, "domain": "service.gov.uk", "state": "html", "status": 200, "robotsTxt": "https://www.gov.uk/", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/service.gov.uk" }, { "rank": 522, "domain": "house.gov", "state": "found", "status": 200, "robotsTxt": "https://www.house.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/house.gov" }, { "rank": 524, "domain": "spiegel.de", "state": "found", "status": 200, "robotsTxt": "https://www.spiegel.de/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "OAI-SearchBot", "Claude-SearchBot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "MistralAI-User", "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/spiegel.de" }, { "rank": 525, "domain": "freebsd.org", "state": "found", "status": 200, "robotsTxt": "https://www.freebsd.org/robots.txt", "blocked": [], "partlyBlocked": [ "GPTBot", "CCBot", "AhrefsBot", "SemrushBot", "DotBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/freebsd.org" }, { "rank": 526, "domain": "pwc.com", "state": "refused", "status": 403, "robotsTxt": "https://www.pwc.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/pwc.com" }, { "rank": 527, "domain": "laravel.com", "state": "found", "status": 200, "robotsTxt": "https://laravel.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/laravel.com" }, { "rank": 531, "domain": "fedoraproject.org", "state": "found", "status": 200, "robotsTxt": "https://fedoraproject.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/fedoraproject.org" }, { "rank": 532, "domain": "snapchat.com", "state": "found", "status": 200, "robotsTxt": "https://www.snapchat.com/robots.txt", "blocked": [ "PetalBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "Meta-ExternalFetcher" ], "partlyBlocked": [ "Googlebot", "Applebot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "ChatGPT-User", "Claude-User", "Perplexity-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/snapchat.com" }, { "rank": 533, "domain": "airbnb.com", "state": "found", "status": 200, "robotsTxt": "https://www.airbnb.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "Applebot-Extended" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Meta-ExternalAgent", "ChatGPT-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/airbnb.com" }, { "rank": 534, "domain": "telegra.ph", "state": "none", "status": 404, "robotsTxt": "https://telegra.ph/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/telegra.ph" }, { "rank": 535, "domain": "clevelandclinic.org", "state": "found", "status": 200, "robotsTxt": "https://my.clevelandclinic.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/clevelandclinic.org" }, { "rank": 538, "domain": "npmjs.com", "state": "refused", "status": 403, "robotsTxt": "https://www.npmjs.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/npmjs.com" }, { "rank": 539, "domain": "generatepress.com", "state": "found", "status": 200, "robotsTxt": "https://generatepress.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/generatepress.com" }, { "rank": 541, "domain": "nodejs.org", "state": "found", "status": 200, "robotsTxt": "https://nodejs.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/nodejs.org" }, { "rank": 542, "domain": "census.gov", "state": "found", "status": 200, "robotsTxt": "https://www.census.gov/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/census.gov" }, { "rank": 543, "domain": "web.dev", "state": "found", "status": 200, "robotsTxt": "https://web.dev/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/web.dev" }, { "rank": 544, "domain": "siemens.com", "state": "found", "status": 200, "robotsTxt": "https://www.siemens.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/siemens.com" }, { "rank": 546, "domain": "energy.gov", "state": "found", "status": 200, "robotsTxt": "https://www.energy.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/energy.gov" }, { "rank": 547, "domain": "buzzfeed.com", "state": "found", "status": 200, "robotsTxt": "https://www.buzzfeed.com/robots.txt", "blocked": [ "PerplexityBot", "DuckAssistBot", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Perplexity-User", "Meta-ExternalFetcher", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/buzzfeed.com" }, { "rank": 548, "domain": "opensource.org", "state": "found", "status": 200, "robotsTxt": "https://opensource.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/opensource.org" }, { "rank": 549, "domain": "vox.com", "state": "found", "status": 200, "robotsTxt": "https://www.vox.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/vox.com" }, { "rank": 551, "domain": "box.com", "state": "found", "status": 200, "robotsTxt": "https://www.box.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/box.com" }, { "rank": 552, "domain": "moodle.com", "state": "error", "status": 520, "robotsTxt": "https://www.moodle.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/moodle.com" }, { "rank": 554, "domain": "ok.ru", "state": "found", "status": 200, "robotsTxt": "https://ok.ru/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ok.ru" }, { "rank": 556, "domain": "disqus.com", "state": "found", "status": 200, "robotsTxt": "https://disqus.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/disqus.com" }, { "rank": 557, "domain": "nyu.edu", "state": "none", "status": 405, "robotsTxt": "https://www.nyu.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nyu.edu" }, { "rank": 558, "domain": "cctv.com", "state": "found", "status": 200, "robotsTxt": "https://www.cctv.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cctv.com" }, { "rank": 559, "domain": "ionos.de", "state": "found", "status": 200, "robotsTxt": "https://www.ionos.de/robots.txt", "blocked": [ "Bingbot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ionos.de" }, { "rank": 561, "domain": "strato.de", "state": "found", "status": 200, "robotsTxt": "https://www.strato.de/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/strato.de" }, { "rank": 562, "domain": "jquery.com", "state": "found", "status": 200, "robotsTxt": "https://jquery.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/jquery.com" }, { "rank": 565, "domain": "marketwatch.com", "state": "found", "status": 200, "robotsTxt": "https://www.marketwatch.com/robots.txt", "blocked": [ "YandexBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "PetalBot", "Baiduspider", "OAI-SearchBot", "GPTBot", "GoogleOther", "ChatGPT-User", "AhrefsBot", "SemrushBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/marketwatch.com" }, { "rank": 566, "domain": "bigcartel.com", "state": "found", "status": 200, "robotsTxt": "https://www.bigcartel.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bigcartel.com" }, { "rank": 568, "domain": "inc.com", "state": "found", "status": 200, "robotsTxt": "https://www.inc.com/robots.txt", "blocked": [ "PerplexityBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Bytespider", "MistralAI-User" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "Baiduspider", "OAI-SearchBot", "Claude-SearchBot", "Google-Extended", "Meta-ExternalAgent", "ChatGPT-User", "Claude-User", "Perplexity-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/inc.com" }, { "rank": 569, "domain": "kde.org", "state": "found", "status": 200, "robotsTxt": "https://kde.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/kde.org" }, { "rank": 570, "domain": "variety.com", "state": "found", "status": 200, "robotsTxt": "https://variety.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/variety.com" }, { "rank": 571, "domain": "douyin.com", "state": "found", "status": 200, "robotsTxt": "https://www.douyin.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "GPTBot", "Google-Extended", "Bytespider", "ChatGPT-User", "Perplexity-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/douyin.com" }, { "rank": 572, "domain": "senate.gov", "state": "html", "status": 200, "robotsTxt": "https://www.senate.gov/", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/senate.gov" }, { "rank": 573, "domain": "imf.org", "state": "none", "status": 404, "robotsTxt": "https://www.imf.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/imf.org" }, { "rank": 575, "domain": "gallup.com", "state": "found", "status": 200, "robotsTxt": "https://www.gallup.com/robots.txt", "blocked": [ "PetalBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gallup.com" }, { "rank": 576, "domain": "congress.gov", "state": "found", "status": 200, "robotsTxt": "https://www.congress.gov/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/congress.gov" }, { "rank": 577, "domain": "cnil.fr", "state": "found", "status": 200, "robotsTxt": "https://cnil.fr/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cnil.fr" }, { "rank": 579, "domain": "ucsd.edu", "state": "found", "status": 200, "robotsTxt": "https://ucsd.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ucsd.edu" }, { "rank": 580, "domain": "medicalnewstoday.com", "state": "found", "status": 200, "robotsTxt": "https://www.medicalnewstoday.com/robots.txt", "blocked": [ "Amazonbot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/medicalnewstoday.com" }, { "rank": 581, "domain": "nejm.org", "state": "found", "status": 200, "robotsTxt": "https://www.nejm.org/robots.txt", "blocked": [ "PerplexityBot", "GPTBot", "Google-Extended", "CCBot", "ChatGPT-User", "AhrefsBot", "SemrushBot" ], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/nejm.org" }, { "rank": 582, "domain": "digg.com", "state": "found", "status": 200, "robotsTxt": "https://digg.com/robots.txt", "blocked": [], "partlyBlocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "ClaudeBot", "Google-Extended", "Applebot-Extended", "ChatGPT-User", "Claude-User", "Perplexity-User" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/digg.com" }, { "rank": 583, "domain": "purdue.edu", "state": "found", "status": 200, "robotsTxt": "https://purdue.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/purdue.edu" }, { "rank": 587, "domain": "constantcontact.com", "state": "found", "status": 200, "robotsTxt": "https://www.constantcontact.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/constantcontact.com" }, { "rank": 588, "domain": "barnesandnoble.com", "state": "found", "status": 200, "robotsTxt": "https://www.barnesandnoble.com/robots.txt", "blocked": [], "partlyBlocked": [ "AhrefsBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/barnesandnoble.com" }, { "rank": 590, "domain": "steamcommunity.com", "state": "found", "status": 200, "robotsTxt": "https://steamcommunity.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/steamcommunity.com" }, { "rank": 591, "domain": "hatena.ne.jp", "state": "found", "status": 200, "robotsTxt": "https://www.hatena.ne.jp/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/hatena.ne.jp" }, { "rank": 593, "domain": "techradar.com", "state": "found", "status": 200, "robotsTxt": "https://www.techradar.com/robots.txt", "blocked": [ "Applebot", "Amazonbot", "YouBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/techradar.com" }, { "rank": 594, "domain": "perl.org", "state": "found", "status": 200, "robotsTxt": "https://www.perl.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/perl.org" }, { "rank": 596, "domain": "java.com", "state": "found", "status": 200, "robotsTxt": "https://www.java.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/java.com" }, { "rank": 597, "domain": "rakuten.co.jp", "state": "html", "status": 200, "robotsTxt": "https://www.rakuten.co.jp/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/rakuten.co.jp" }, { "rank": 598, "domain": "technologyreview.com", "state": "found", "status": 200, "robotsTxt": "https://www.technologyreview.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/technologyreview.com" }, { "rank": 599, "domain": "fliphtml5.com", "state": "found", "status": 200, "robotsTxt": "https://fliphtml5.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "Amazonbot", "YouBot", "CCBot", "Meta-ExternalAgent", "Bytespider", "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/fliphtml5.com" }, { "rank": 600, "domain": "photobucket.com", "state": "found", "status": 200, "robotsTxt": "https://photobucket.com/robots.txt", "blocked": [ "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/photobucket.com" }, { "rank": 601, "domain": "trello.com", "state": "found", "status": 200, "robotsTxt": "https://trello.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/trello.com" }, { "rank": 602, "domain": "namepros.com", "state": "found", "status": 200, "robotsTxt": "https://www.namepros.com/robots.txt", "blocked": [], "partlyBlocked": [ "AhrefsBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/namepros.com" }, { "rank": 603, "domain": "politico.com", "state": "refused", "status": 403, "robotsTxt": "https://www.politico.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/politico.com" }, { "rank": 604, "domain": "apachefriends.org", "state": "found", "status": 200, "robotsTxt": "https://www.apachefriends.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/apachefriends.org" }, { "rank": 605, "domain": "amd.com", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/amd.com" }, { "rank": 606, "domain": "cve.org", "state": "found", "status": 200, "robotsTxt": "https://www.cve.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cve.org" }, { "rank": 607, "domain": "matomo.org", "state": "found", "status": 200, "robotsTxt": "https://matomo.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/matomo.org" }, { "rank": 609, "domain": "entrepreneur.com", "state": "found", "status": 200, "robotsTxt": "https://www.entrepreneur.com/robots.txt", "blocked": [ "Meta-ExternalAgent", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/entrepreneur.com" }, { "rank": 610, "domain": "qz.com", "state": "found", "status": 200, "robotsTxt": "https://qz.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/qz.com" }, { "rank": 611, "domain": "scmp.com", "state": "found", "status": 200, "robotsTxt": "https://www.scmp.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/scmp.com" }, { "rank": 612, "domain": "zend.com", "state": "found", "status": 200, "robotsTxt": "https://www.zend.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/zend.com" }, { "rank": 613, "domain": "ufl.edu", "state": "found", "status": 200, "robotsTxt": "https://www.ufl.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ufl.edu" }, { "rank": 614, "domain": "so.com", "state": "found", "status": 200, "robotsTxt": "https://www.so.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/so.com" }, { "rank": 615, "domain": "claude.ai", "state": "found", "status": 200, "robotsTxt": "https://claude.ai/robots.txt", "blocked": [ "OAI-SearchBot", "GPTBot", "Google-Extended", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/claude.ai" }, { "rank": 616, "domain": "slate.com", "state": "found", "status": 200, "robotsTxt": "https://slate.com/robots.txt", "blocked": [ "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [ "AhrefsBot", "SemrushBot", "DotBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/slate.com" }, { "rank": 617, "domain": "drupal.org", "state": "found", "status": 200, "robotsTxt": "https://www.drupal.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/drupal.org" }, { "rank": 618, "domain": "chinadaily.com.cn", "state": "none", "status": 404, "robotsTxt": "https://www.chinadaily.com.cn/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/chinadaily.com.cn" }, { "rank": 620, "domain": "earthlink.net", "state": "found", "status": 200, "robotsTxt": "https://www.earthlink.net/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/earthlink.net" }, { "rank": 621, "domain": "duckduckgo.com", "state": "found", "status": 200, "robotsTxt": "https://duckduckgo.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/duckduckgo.com" }, { "rank": 622, "domain": "lww.com", "state": "html", "status": 200, "robotsTxt": "https://www.ovid.com/browse", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/lww.com" }, { "rank": 623, "domain": "udemy.com", "state": "refused", "status": 403, "robotsTxt": "https://udemy.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/udemy.com" }, { "rank": 625, "domain": "duke.edu", "state": "found", "status": 200, "robotsTxt": "https://www.duke.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/duke.edu" }, { "rank": 628, "domain": "usc.edu", "state": "found", "status": 200, "robotsTxt": "https://usc.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/usc.edu" }, { "rank": 629, "domain": "vmware.com", "state": "found", "status": 200, "robotsTxt": "https://www.vmware.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/vmware.com" }, { "rank": 630, "domain": "sap.com", "state": "found", "status": 200, "robotsTxt": "https://www.sap.com/robots.txt", "blocked": [ "PetalBot", "YandexBot", "DuckAssistBot", "YouBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "GoogleOther", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/sap.com" }, { "rank": 631, "domain": "meta.com", "state": "found", "status": 200, "robotsTxt": "https://www.meta.com/robots.txt", "blocked": [ "PetalBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/meta.com" }, { "rank": 632, "domain": "rollingstone.com", "state": "found", "status": 200, "robotsTxt": "https://www.rollingstone.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/rollingstone.com" }, { "rank": 633, "domain": "coe.int", "state": "refused", "status": 403, "robotsTxt": "https://www.coe.int/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/coe.int" }, { "rank": 635, "domain": "pcmag.com", "state": "found", "status": 200, "robotsTxt": "https://www.pcmag.com/robots.txt", "blocked": [ "Applebot", "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/pcmag.com" }, { "rank": 637, "domain": "shutterstock.com", "state": "found", "status": 200, "robotsTxt": "https://www.shutterstock.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "CCBot", "Meta-ExternalAgent" ], "partlyBlocked": [ "Bingbot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/shutterstock.com" }, { "rank": 638, "domain": "ifeng.com", "state": "found", "status": 200, "robotsTxt": "https://www.ifeng.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/ifeng.com" }, { "rank": 639, "domain": "mail.ru", "state": "found", "status": 200, "robotsTxt": "https://mail.ru/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/mail.ru" }, { "rank": 640, "domain": "semrush.com", "state": "found", "status": 200, "robotsTxt": "https://www.semrush.com/robots.txt", "blocked": [], "partlyBlocked": [ "Bingbot" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/semrush.com" }, { "rank": 641, "domain": "umd.edu", "state": "found", "status": 200, "robotsTxt": "https://umd.edu/robots.txt", "blocked": [ "PerplexityBot", "GPTBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/umd.edu" }, { "rank": 642, "domain": "ccc.de", "state": "found", "status": 200, "robotsTxt": "https://www.ccc.de/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ccc.de" }, { "rank": 643, "domain": "gutenberg.org", "state": "found", "status": 200, "robotsTxt": "https://www.gutenberg.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gutenberg.org" }, { "rank": 644, "domain": "qualtrics.com", "state": "found", "status": 200, "robotsTxt": "https://www.qualtrics.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/qualtrics.com" }, { "rank": 646, "domain": "utoronto.ca", "state": "refused", "status": 403, "robotsTxt": "https://utoronto.ca/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/utoronto.ca" }, { "rank": 647, "domain": "ny.gov", "state": "found", "status": 200, "robotsTxt": "https://www.ny.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ny.gov" }, { "rank": 648, "domain": "timeanddate.com", "state": "found", "status": 200, "robotsTxt": "https://www.timeanddate.com/robots.txt", "blocked": [ "PerplexityBot", "Amazonbot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [ "AhrefsBot", "DotBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/timeanddate.com" }, { "rank": 649, "domain": "ycombinator.com", "state": "found", "status": 200, "robotsTxt": "https://www.ycombinator.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ycombinator.com" }, { "rank": 651, "domain": "autodesk.com", "state": "refused", "status": 403, "robotsTxt": "https://www.autodesk.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/autodesk.com" }, { "rank": 652, "domain": "ucl.ac.uk", "state": "refused", "status": 403, "robotsTxt": "https://www.ucl.ac.uk/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/ucl.ac.uk" }, { "rank": 653, "domain": "hetzner.com", "state": "found", "status": 200, "robotsTxt": "https://www.hetzner.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/hetzner.com" }, { "rank": 654, "domain": "fifa.com", "state": "found", "status": 200, "robotsTxt": "https://www.fifa.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/fifa.com" }, { "rank": 656, "domain": "mixcloud.com", "state": "found", "status": 200, "robotsTxt": "https://www.mixcloud.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/mixcloud.com" }, { "rank": 657, "domain": "zoho.com", "state": "found", "status": 200, "robotsTxt": "https://www.zoho.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/zoho.com" }, { "rank": 658, "domain": "slashdot.org", "state": "found", "status": 200, "robotsTxt": "https://slashdot.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/slashdot.org" }, { "rank": 659, "domain": "axios.com", "state": "found", "status": 200, "robotsTxt": "https://www.axios.com/robots.txt", "blocked": [ "Amazonbot", "CCBot", "Bytespider" ], "partlyBlocked": [ "Google-Extended" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/axios.com" }, { "rank": 660, "domain": "fc2.com", "state": "found", "status": 200, "robotsTxt": "https://fc2.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/fc2.com" }, { "rank": 661, "domain": "parallels.com", "state": "found", "status": 200, "robotsTxt": "https://www.parallels.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/parallels.com" }, { "rank": 662, "domain": "woocommerce.com", "state": "found", "status": 200, "robotsTxt": "https://woocommerce.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/woocommerce.com" }, { "rank": 664, "domain": "ey.com", "state": "found", "status": 200, "robotsTxt": "https://www.ey.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ey.com" }, { "rank": 665, "domain": "semanticscholar.org", "state": "found", "status": 200, "robotsTxt": "https://www.semanticscholar.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/semanticscholar.org" }, { "rank": 667, "domain": "qcloud.com", "state": "found", "status": 200, "robotsTxt": "https://qcloud.com/robots.txt", "blocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/qcloud.com" }, { "rank": 668, "domain": "icann.org", "state": "found", "status": 200, "robotsTxt": "https://www.icann.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/icann.org" }, { "rank": 669, "domain": "digitalocean.com", "state": "found", "status": 200, "robotsTxt": "https://www.digitalocean.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/digitalocean.com" }, { "rank": 670, "domain": "newscientist.com", "state": "found", "status": 200, "robotsTxt": "https://www.newscientist.com/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/newscientist.com" }, { "rank": 671, "domain": "ilo.org", "state": "found", "status": 200, "robotsTxt": "https://www.ilo.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ilo.org" }, { "rank": 672, "domain": "ucdavis.edu", "state": "found", "status": 200, "robotsTxt": "https://www.ucdavis.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ucdavis.edu" }, { "rank": 673, "domain": "cell.com", "state": "found", "status": 200, "robotsTxt": "https://www.cell.com/robots.txt", "blocked": [ "PerplexityBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "MistralAI-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cell.com" }, { "rank": 676, "domain": "unc.edu", "state": "found", "status": 200, "robotsTxt": "https://www.unc.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/unc.edu" }, { "rank": 677, "domain": "indiegogo.com", "state": "refused", "status": 403, "robotsTxt": "https://www.indiegogo.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/indiegogo.com" }, { "rank": 678, "domain": "livescience.com", "state": "found", "status": 200, "robotsTxt": "https://www.livescience.com/robots.txt", "blocked": [ "Applebot", "Amazonbot", "YouBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/livescience.com" }, { "rank": 680, "domain": "broadcom.com", "state": "found", "status": 200, "robotsTxt": "https://www.broadcom.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/broadcom.com" }, { "rank": 681, "domain": "sfgate.com", "state": "found", "status": 200, "robotsTxt": "https://www.sfgate.com/robots.txt", "blocked": [ "PerplexityBot", "DuckAssistBot", "YouBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Bytespider", "Claude-User", "Perplexity-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/sfgate.com" }, { "rank": 683, "domain": "nikkei.com", "state": "found", "status": 200, "robotsTxt": "https://www.nikkei.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nikkei.com" }, { "rank": 684, "domain": "mediafire.com", "state": "found", "status": 200, "robotsTxt": "https://mediafire.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/mediafire.com" }, { "rank": 686, "domain": "illinois.edu", "state": "found", "status": 200, "robotsTxt": "https://illinois.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/illinois.edu" }, { "rank": 687, "domain": "getpocket.com", "state": "found", "status": 200, "robotsTxt": "https://getpocket.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/getpocket.com" }, { "rank": 688, "domain": "espn.com", "state": "found", "status": 200, "robotsTxt": "https://www.espn.com/robots.txt", "blocked": [ "GPTBot", "Google-Extended", "CCBot", "Bytespider", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/espn.com" }, { "rank": 689, "domain": "venturebeat.com", "state": "found", "status": 200, "robotsTxt": "https://venturebeat.com/robots.txt", "blocked": [ "YandexBot", "Amazonbot", "CCBot", "Applebot-Extended", "Bytespider", "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "PerplexityBot", "GPTBot", "ClaudeBot", "Google-Extended" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/venturebeat.com" }, { "rank": 690, "domain": "sky.com", "state": "found", "status": 200, "robotsTxt": "https://www.sky.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/sky.com" }, { "rank": 691, "domain": "thesun.co.uk", "state": "found", "status": 200, "robotsTxt": "https://www.thesun.co.uk/robots.txt", "blocked": [ "PetalBot", "YandexBot", "Baiduspider", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "OAI-SearchBot", "GPTBot", "GoogleOther", "ChatGPT-User", "SemrushBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/thesun.co.uk" }, { "rank": 694, "domain": "mirror.co.uk", "state": "found", "status": 200, "robotsTxt": "https://www.mirror.co.uk/robots.txt", "blocked": [ "Bingbot", "OAI-SearchBot", "PerplexityBot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/mirror.co.uk" }, { "rank": 696, "domain": "howstuffworks.com", "state": "found", "status": 200, "robotsTxt": "https://animals.howstuffworks.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/howstuffworks.com" }, { "rank": 697, "domain": "playstation.com", "state": "found", "status": 200, "robotsTxt": "https://www.playstation.com/robots.txt", "blocked": [ "PetalBot" ], "partlyBlocked": [ "Bingbot", "YandexBot", "AhrefsBot" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/playstation.com" }, { "rank": 699, "domain": "msu.edu", "state": "found", "status": 200, "robotsTxt": "https://msu.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/msu.edu" }, { "rank": 700, "domain": "altervista.org", "state": "html", "status": 200, "robotsTxt": "https://en.altervista.org/", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/altervista.org" }, { "rank": 701, "domain": "yandex.com", "state": "found", "status": 200, "robotsTxt": "https://yandex.com/robots.txt", "blocked": [], "partlyBlocked": [ "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/yandex.com" }, { "rank": 702, "domain": "lenovo.com", "state": "refused", "status": 403, "robotsTxt": "https://www.lenovo.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/lenovo.com" }, { "rank": 703, "domain": "smh.com.au", "state": "found", "status": 200, "robotsTxt": "https://www.smh.com.au/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/smh.com.au" }, { "rank": 704, "domain": "curl.se", "state": "found", "status": 200, "robotsTxt": "https://curl.se/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/curl.se" }, { "rank": 705, "domain": "mastodon.social", "state": "found", "status": 200, "robotsTxt": "https://mastodon.social/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/mastodon.social" }, { "rank": 706, "domain": "itch.io", "state": "found", "status": 200, "robotsTxt": "https://itch.io/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/itch.io" }, { "rank": 707, "domain": "archives.gov", "state": "found", "status": 200, "robotsTxt": "https://www.archives.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/archives.gov" }, { "rank": 708, "domain": "iea.org", "state": "found", "status": 200, "robotsTxt": "https://www.iea.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/iea.org" }, { "rank": 709, "domain": "undp.org", "state": "found", "status": 200, "robotsTxt": "https://www.undp.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/undp.org" }, { "rank": 710, "domain": "ubc.ca", "state": "found", "status": 200, "robotsTxt": "https://www.ubc.ca/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ubc.ca" }, { "rank": 714, "domain": "claude.com", "state": "found", "status": 200, "robotsTxt": "https://claude.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/claude.com" }, { "rank": 715, "domain": "ethz.ch", "state": "none", "status": 404, "robotsTxt": "https://ethz.ch/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ethz.ch" }, { "rank": 716, "domain": "smithsonianmag.com", "state": "html", "status": 200, "robotsTxt": "https://www.smithsonianmag.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/smithsonianmag.com" }, { "rank": 717, "domain": "hollywoodreporter.com", "state": "found", "status": 200, "robotsTxt": "https://www.hollywoodreporter.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/hollywoodreporter.com" }, { "rank": 718, "domain": "pypi.org", "state": "found", "status": 200, "robotsTxt": "https://pypi.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/pypi.org" }, { "rank": 719, "domain": "iop.org", "state": "found", "status": 200, "robotsTxt": "https://iop.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/iop.org" }, { "rank": 720, "domain": "caniuse.com", "state": "html", "status": 200, "robotsTxt": "https://caniuse.com/?search=robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/caniuse.com" }, { "rank": 721, "domain": "phys.org", "state": "found", "status": 200, "robotsTxt": "https://phys.org/robots.txt", "blocked": [], "partlyBlocked": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Bytespider", "AhrefsBot" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/phys.org" }, { "rank": 722, "domain": "mi.com", "state": "refused", "status": 403, "robotsTxt": "https://www.mi.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/mi.com" }, { "rank": 723, "domain": "proofpoint.com", "state": "found", "status": 200, "robotsTxt": "https://www.proofpoint.com/robots.txt", "blocked": [ "Meta-ExternalAgent" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/proofpoint.com" }, { "rank": 725, "domain": "hostinger.com", "state": "found", "status": 200, "robotsTxt": "https://www.hostinger.com/robots.txt", "blocked": [ "Bytespider" ], "partlyBlocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/hostinger.com" }, { "rank": 726, "domain": "medlineplus.gov", "state": "found", "status": 200, "robotsTxt": "https://medlineplus.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/medlineplus.gov" }, { "rank": 727, "domain": "fcc.gov", "state": "refused", "status": 403, "robotsTxt": "https://fcc.gov/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/fcc.gov" }, { "rank": 728, "domain": "ign.com", "state": "found", "status": 200, "robotsTxt": "https://www.ign.com/robots.txt", "blocked": [ "Applebot", "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ign.com" }, { "rank": 730, "domain": "wipo.int", "state": "found", "status": 200, "robotsTxt": "https://www.wipo.int/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/wipo.int" }, { "rank": 731, "domain": "cpan.org", "state": "found", "status": 200, "robotsTxt": "https://www.cpan.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cpan.org" }, { "rank": 732, "domain": "news.cn", "state": "found", "status": 200, "robotsTxt": "https://www.news.cn/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/news.cn" }, { "rank": 733, "domain": "colorado.edu", "state": "found", "status": 200, "robotsTxt": "https://www.colorado.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/colorado.edu" }, { "rank": 734, "domain": "northwestern.edu", "state": "found", "status": 200, "robotsTxt": "https://www.northwestern.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/northwestern.edu" }, { "rank": 735, "domain": "asu.edu", "state": "found", "status": 200, "robotsTxt": "https://www.asu.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/asu.edu" }, { "rank": 736, "domain": "thehill.com", "state": "found", "status": 200, "robotsTxt": "https://thehill.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/thehill.com" }, { "rank": 737, "domain": "chicagotribune.com", "state": "found", "status": 200, "robotsTxt": "https://www.chicagotribune.com/robots.txt", "blocked": [ "Applebot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/chicagotribune.com" }, { "rank": 738, "domain": "history.com", "state": "found", "status": 200, "robotsTxt": "https://www.history.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/history.com" }, { "rank": 739, "domain": "csdn.net", "state": "found", "status": 200, "robotsTxt": "https://www.csdn.net/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/csdn.net" }, { "rank": 741, "domain": "tmall.com", "state": "found", "status": 200, "robotsTxt": "https://www.tmall.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/tmall.com" }, { "rank": 744, "domain": "heise.de", "state": "none", "status": 410, "robotsTxt": "https://heise.de/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/heise.de" }, { "rank": 745, "domain": "laracasts.com", "state": "found", "status": 200, "robotsTxt": "https://laracasts.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/laracasts.com" }, { "rank": 746, "domain": "imagemagick.org", "state": "found", "status": 200, "robotsTxt": "https://imagemagick.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/imagemagick.org" }, { "rank": 747, "domain": "haxx.se", "state": "none", "status": 404, "robotsTxt": "https://haxx.se/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/haxx.se" }, { "rank": 749, "domain": "ohchr.org", "state": "found", "status": 200, "robotsTxt": "https://www.ohchr.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ohchr.org" }, { "rank": 751, "domain": "theglobeandmail.com", "state": "found", "status": 200, "robotsTxt": "https://www.theglobeandmail.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/theglobeandmail.com" }, { "rank": 752, "domain": "calameo.com", "state": "found", "status": 200, "robotsTxt": "https://www.calameo.com/robots.txt", "blocked": [ "GoogleOther" ], "partlyBlocked": [ "Amazonbot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/calameo.com" }, { "rank": 754, "domain": "thetimes.co.uk", "state": "found", "status": 200, "robotsTxt": "https://www.thetimes.co.uk/robots.txt", "blocked": [ "PetalBot", "YandexBot", "Baiduspider", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "OAI-SearchBot", "GPTBot", "GoogleOther", "ChatGPT-User", "SemrushBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/thetimes.co.uk" }, { "rank": 755, "domain": "nike.com", "state": "found", "status": 200, "robotsTxt": "https://www.nike.com/robots.txt", "blocked": [ "Baiduspider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nike.com" }, { "rank": 756, "domain": "garmin.com", "state": "found", "status": 200, "robotsTxt": "https://www.garmin.com/robots.txt", "blocked": [ "PerplexityBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/garmin.com" }, { "rank": 757, "domain": "bizjournals.com", "state": "found", "status": 200, "robotsTxt": "https://www.bizjournals.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bizjournals.com" }, { "rank": 758, "domain": "abcnews.com", "state": "found", "status": 200, "robotsTxt": "https://abcnews.com/robots.txt", "blocked": [ "GPTBot", "Google-Extended", "CCBot", "Bytespider", "ChatGPT-User" ], "partlyBlocked": [ "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/abcnews.com" }, { "rank": 760, "domain": "treasury.gov", "state": "html", "status": 200, "robotsTxt": "https://home.treasury.gov/", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/treasury.gov" }, { "rank": 762, "domain": "openldap.org", "state": "found", "status": 200, "robotsTxt": "https://openldap.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/openldap.org" }, { "rank": 764, "domain": "arizona.edu", "state": "found", "status": 200, "robotsTxt": "https://www.arizona.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/arizona.edu" }, { "rank": 765, "domain": "fontawesome.com", "state": "found", "status": 200, "robotsTxt": "https://fontawesome.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/fontawesome.com" }, { "rank": 767, "domain": "kpmg.com", "state": "found", "status": 200, "robotsTxt": "https://kpmg.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/kpmg.com" }, { "rank": 769, "domain": "accor.com", "state": "html", "status": 200, "robotsTxt": "https://group.accor.com/en", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/accor.com" }, { "rank": 770, "domain": "joomla.org", "state": "found", "status": 200, "robotsTxt": "https://www.joomla.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/joomla.org" }, { "rank": 771, "domain": "man7.org", "state": "found", "status": 200, "robotsTxt": "https://man7.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/man7.org" }, { "rank": 772, "domain": "ameblo.jp", "state": "found", "status": 200, "robotsTxt": "https://ameblo.jp/robots.txt", "blocked": [ "Bingbot" ], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/ameblo.jp" }, { "rank": 773, "domain": "fbi.gov", "state": "found", "status": 200, "robotsTxt": "https://www.fbi.gov/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/fbi.gov" }, { "rank": 774, "domain": "podbean.com", "state": "found", "status": 200, "robotsTxt": "https://www.podbean.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/podbean.com" }, { "rank": 775, "domain": "virginia.edu", "state": "found", "status": 200, "robotsTxt": "https://www.virginia.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/virginia.edu" }, { "rank": 777, "domain": "binance.com", "state": "found", "status": 200, "robotsTxt": "https://www.binance.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "Baiduspider" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/binance.com" }, { "rank": 778, "domain": "notion.so", "state": "found", "status": 200, "robotsTxt": "https://www.notion.so/robots.txt", "blocked": [ "Amazonbot", "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/notion.so" }, { "rank": 780, "domain": "france24.com", "state": "found", "status": 200, "robotsTxt": "https://www.france24.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/france24.com" }, { "rank": 781, "domain": "deezer.com", "state": "found", "status": 200, "robotsTxt": "https://www.deezer.com/robots.txt", "blocked": [], "partlyBlocked": [ "Applebot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/deezer.com" }, { "rank": 782, "domain": "khanacademy.org", "state": "html", "status": 200, "robotsTxt": "https://www.khanacademy.org/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/khanacademy.org" }, { "rank": 784, "domain": "owasp.org", "state": "found", "status": 200, "robotsTxt": "https://owasp.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/owasp.org" }, { "rank": 785, "domain": "lefigaro.fr", "state": "found", "status": 200, "robotsTxt": "https://www.lefigaro.fr/robots.txt", "blocked": [ "OAI-SearchBot", "Claude-SearchBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "ChatGPT-User", "Claude-User", "MistralAI-User", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/lefigaro.fr" }, { "rank": 787, "domain": "heart.org", "state": "refused", "status": 403, "robotsTxt": "https://www.heart.org/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/heart.org" }, { "rank": 788, "domain": "ihg.com", "state": "refused", "status": 403, "robotsTxt": "https://ihg.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/ihg.com" }, { "rank": 789, "domain": "asus.com", "state": "found", "status": 200, "robotsTxt": "https://www.asus.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/asus.com" }, { "rank": 790, "domain": "moz.com", "state": "found", "status": 200, "robotsTxt": "https://moz.com/robots.txt", "blocked": [], "partlyBlocked": [ "GPTBot", "AhrefsBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/moz.com" }, { "rank": 791, "domain": "about.me", "state": "found", "status": 200, "robotsTxt": "https://about.me/robots.txt", "blocked": [ "SemrushBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/about.me" }, { "rank": 792, "domain": "cygwin.com", "state": "found", "status": 200, "robotsTxt": "https://cygwin.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cygwin.com" }, { "rank": 794, "domain": "jetbrains.com", "state": "found", "status": 200, "robotsTxt": "https://www.jetbrains.com/robots.txt", "blocked": [], "partlyBlocked": [ "Baiduspider" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/jetbrains.com" }, { "rank": 796, "domain": "uspto.gov", "state": "found", "status": 200, "robotsTxt": "https://www.uspto.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/uspto.gov" }, { "rank": 798, "domain": "uber.com", "state": "found", "status": 200, "robotsTxt": "https://www.uber.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/uber.com" }, { "rank": 799, "domain": "mastercard.com", "state": "refused", "status": 403, "robotsTxt": "https://www.mastercard.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/mastercard.com" }, { "rank": 800, "domain": "dhs.gov", "state": "found", "status": 200, "robotsTxt": "https://www.dhs.gov/robots.txt", "blocked": [ "PetalBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/dhs.gov" }, { "rank": 801, "domain": "uol.com.br", "state": "found", "status": 200, "robotsTxt": "https://www.uol.com.br/robots.txt", "blocked": [ "Applebot", "PetalBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/uol.com.br" }, { "rank": 802, "domain": "weather.com", "state": "found", "status": 200, "robotsTxt": "https://weather.com/robots.txt", "blocked": [ "Amazonbot", "YouBot", "CCBot", "GoogleOther", "Bytespider" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/weather.com" }, { "rank": 804, "domain": "dot.gov", "state": "refused", "status": 403, "robotsTxt": "https://www.dot.gov/", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/dot.gov" }, { "rank": 805, "domain": "evernote.com", "state": "found", "status": 200, "robotsTxt": "https://evernote.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/evernote.com" }, { "rank": 806, "domain": "verisign.com", "state": "found", "status": 200, "robotsTxt": "https://www.verisign.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/verisign.com" }, { "rank": 807, "domain": "cancer.gov", "state": "found", "status": 200, "robotsTxt": "https://www.cancer.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cancer.gov" }, { "rank": 809, "domain": "va.gov", "state": "found", "status": 200, "robotsTxt": "https://www.va.gov/robots.txt", "blocked": [ "Amazonbot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/va.gov" }, { "rank": 812, "domain": "kaspersky.com", "state": "found", "status": 200, "robotsTxt": "https://www.kaspersky.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/kaspersky.com" }, { "rank": 813, "domain": "brookings.edu", "state": "found", "status": 200, "robotsTxt": "https://www.brookings.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/brookings.edu" }, { "rank": 815, "domain": "ed.ac.uk", "state": "found", "status": 200, "robotsTxt": "https://www.ed.ac.uk/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ed.ac.uk" }, { "rank": 816, "domain": "accenture.com", "state": "found", "status": 200, "robotsTxt": "https://www.accenture.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/accenture.com" }, { "rank": 817, "domain": "webex.com", "state": "refused", "status": 403, "robotsTxt": "https://www.webex.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/webex.com" }, { "rank": 818, "domain": "wa.gov", "state": "found", "status": 200, "robotsTxt": "https://wa.gov/robots.txt", "blocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/wa.gov" }, { "rank": 819, "domain": "mpg.de", "state": "found", "status": 200, "robotsTxt": "https://www.mpg.de/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/mpg.de" }, { "rank": 822, "domain": "people.com", "state": "found", "status": 200, "robotsTxt": "https://people.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [ "OAI-SearchBot", "GPTBot", "ChatGPT-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/people.com" }, { "rank": 823, "domain": "bitbucket.org", "state": "found", "status": 200, "robotsTxt": "https://bitbucket.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bitbucket.org" }, { "rank": 824, "domain": "globo.com", "state": "found", "status": 200, "robotsTxt": "https://www.globo.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/globo.com" }, { "rank": 825, "domain": "tamu.edu", "state": "found", "status": 200, "robotsTxt": "https://www.tamu.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/tamu.edu" }, { "rank": 826, "domain": "redis.io", "state": "found", "status": 200, "robotsTxt": "https://redis.io/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/redis.io" }, { "rank": 827, "domain": "texas.gov", "state": "found", "status": 200, "robotsTxt": "https://www.texas.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/texas.gov" }, { "rank": 828, "domain": "instructables.com", "state": "found", "status": 200, "robotsTxt": "https://www.instructables.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/instructables.com" }, { "rank": 830, "domain": "esa.int", "state": "found", "status": 200, "robotsTxt": "https://www.esa.int/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/esa.int" }, { "rank": 831, "domain": "nobelprize.org", "state": "found", "status": 200, "robotsTxt": "https://www.nobelprize.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nobelprize.org" }, { "rank": 832, "domain": "figma.com", "state": "found", "status": 200, "robotsTxt": "https://www.figma.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Bytespider", "ChatGPT-User", "Claude-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/figma.com" }, { "rank": 834, "domain": "freedesktop.org", "state": "found", "status": 200, "robotsTxt": "https://freedesktop.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/freedesktop.org" }, { "rank": 835, "domain": "gmail.com", "state": "found", "status": 200, "robotsTxt": "https://gmail.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gmail.com" }, { "rank": 836, "domain": "rutgers.edu", "state": "found", "status": 200, "robotsTxt": "https://www.rutgers.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/rutgers.edu" }, { "rank": 837, "domain": "novell.com", "state": "found", "status": 200, "robotsTxt": "https://www.novell.com/robots.txt", "blocked": [ "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/novell.com" }, { "rank": 838, "domain": "alipay.com", "state": "none", "status": 404, "robotsTxt": "https://alipay.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/alipay.com" }, { "rank": 839, "domain": "usembassy.gov", "state": "refused", "status": 403, "robotsTxt": "https://usembassy.gov/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/usembassy.gov" }, { "rank": 840, "domain": "unep.org", "state": "found", "status": 200, "robotsTxt": "https://www.unep.org/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/unep.org" }, { "rank": 841, "domain": "perplexity.ai", "state": "found", "status": 200, "robotsTxt": "https://www.perplexity.ai/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/perplexity.ai" }, { "rank": 842, "domain": "ahrefs.com", "state": "found", "status": 200, "robotsTxt": "https://ahrefs.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ahrefs.com" }, { "rank": 844, "domain": "olympics.com", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/olympics.com" }, { "rank": 845, "domain": "wireshark.org", "state": "found", "status": 200, "robotsTxt": "https://www.wireshark.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/wireshark.org" }, { "rank": 846, "domain": "wpengine.com", "state": "error", "status": 502, "robotsTxt": "https://wpengine.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/wpengine.com" }, { "rank": 847, "domain": "wikisource.org", "state": "found", "status": 200, "robotsTxt": "https://wikisource.org/robots.txt", "blocked": [], "partlyBlocked": [ "SemrushBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/wikisource.org" }, { "rank": 849, "domain": "gesetze-im-internet.de", "state": "found", "status": 200, "robotsTxt": "https://www.gesetze-im-internet.de/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gesetze-im-internet.de" }, { "rank": 850, "domain": "bitnami.com", "state": "found", "status": 200, "robotsTxt": "https://bitnami.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bitnami.com" }, { "rank": 851, "domain": "osu.edu", "state": "found", "status": 200, "robotsTxt": "https://www.osu.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/osu.edu" }, { "rank": 853, "domain": "bu.edu", "state": "found", "status": 200, "robotsTxt": "https://www.bu.edu/robots.txt", "blocked": [ "GoogleOther" ], "partlyBlocked": [ "Applebot", "YandexBot", "Baiduspider", "Amazonbot", "GPTBot", "ClaudeBot", "CCBot", "Meta-ExternalAgent", "Bytespider" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/bu.edu" }, { "rank": 854, "domain": "envato.com", "state": "found", "status": 200, "robotsTxt": "https://www.envato.com/robots.txt", "blocked": [ "CCBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/envato.com" }, { "rank": 855, "domain": "crunchbase.com", "state": "found", "status": 200, "robotsTxt": "https://www.crunchbase.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/crunchbase.com" }, { "rank": 856, "domain": "speedtest.net", "state": "found", "status": 200, "robotsTxt": "https://www.speedtest.net/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/speedtest.net" }, { "rank": 857, "domain": "usa.gov", "state": "found", "status": 200, "robotsTxt": "https://www.usa.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/usa.gov" }, { "rank": 859, "domain": "1688.com", "state": "found", "status": 200, "robotsTxt": "https://www.1688.com/robots.txt", "blocked": [ "Bingbot" ], "partlyBlocked": [ "Baiduspider" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/1688.com" }, { "rank": 860, "domain": "vatican.va", "state": "found", "status": 200, "robotsTxt": "https://www.vatican.va/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/vatican.va" }, { "rank": 862, "domain": "army.mil", "state": "found", "status": 200, "robotsTxt": "https://www.army.mil/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/army.mil" }, { "rank": 863, "domain": "www.gov.br", "state": "found", "status": 200, "robotsTxt": "https://www.gov.br/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/www.gov.br" }, { "rank": 864, "domain": "centos.org", "state": "found", "status": 200, "robotsTxt": "https://centos.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/centos.org" }, { "rank": 866, "domain": "dol.gov", "state": "found", "status": 200, "robotsTxt": "https://www.dol.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/dol.gov" }, { "rank": 867, "domain": "cancer.org", "state": "found", "status": 200, "robotsTxt": "https://www.cancer.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cancer.org" }, { "rank": 868, "domain": "nasdaq.com", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/nasdaq.com" }, { "rank": 869, "domain": "greenpeace.org", "state": "found", "status": 200, "robotsTxt": "https://www.greenpeace.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/greenpeace.org" }, { "rank": 870, "domain": "express.co.uk", "state": "found", "status": 200, "robotsTxt": "https://express.co.uk/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/express.co.uk" }, { "rank": 871, "domain": "ipcc.ch", "state": "found", "status": 200, "robotsTxt": "https://www.ipcc.ch/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ipcc.ch" }, { "rank": 873, "domain": "utah.edu", "state": "found", "status": 200, "robotsTxt": "https://www.utah.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/utah.edu" }, { "rank": 874, "domain": "fiverr.com", "state": "found", "status": 200, "robotsTxt": "https://www.fiverr.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/fiverr.com" }, { "rank": 875, "domain": "imageshack.us", "state": "found", "status": 200, "robotsTxt": "https://imageshack.us/robots.txt", "blocked": [ "PetalBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [ "Applebot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/imageshack.us" }, { "rank": 877, "domain": "dreamhost.com", "state": "found", "status": 200, "robotsTxt": "https://www.dreamhost.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/dreamhost.com" }, { "rank": 878, "domain": "xbox.com", "state": "found", "status": 200, "robotsTxt": "https://www.xbox.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/xbox.com" }, { "rank": 880, "domain": "ea.com", "state": "found", "status": 200, "robotsTxt": "https://www.ea.com/robots.txt", "blocked": [ "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/ea.com" }, { "rank": 881, "domain": "federalregister.gov", "state": "found", "status": 200, "robotsTxt": "https://www.federalregister.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/federalregister.gov" }, { "rank": 882, "domain": "rutube.ru", "state": "found", "status": 200, "robotsTxt": "https://rutube.ru/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/rutube.ru" }, { "rank": 883, "domain": "gumroad.com", "state": "found", "status": 200, "robotsTxt": "https://gumroad.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/gumroad.com" }, { "rank": 885, "domain": "douban.com", "state": "found", "status": 200, "robotsTxt": "https://www.douban.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/douban.com" }, { "rank": 886, "domain": "nintendo.com", "state": "found", "status": 200, "robotsTxt": "https://www.nintendo.com/robots.txt", "blocked": [ "PetalBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nintendo.com" }, { "rank": 887, "domain": "lua.org", "state": "none", "status": 404, "robotsTxt": "https://www.lua.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/lua.org" }, { "rank": 888, "domain": "toutiao.com", "state": "found", "status": 200, "robotsTxt": "https://www.toutiao.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/toutiao.com" }, { "rank": 889, "domain": "go.dev", "state": "found", "status": 200, "robotsTxt": "https://go.dev/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/go.dev" }, { "rank": 890, "domain": "nhk.or.jp", "state": "found", "status": 200, "robotsTxt": "https://www.nhk.or.jp/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nhk.or.jp" }, { "rank": 891, "domain": "mediawiki.org", "state": "found", "status": 200, "robotsTxt": "https://www.mediawiki.org/robots.txt", "blocked": [], "partlyBlocked": [ "SemrushBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/mediawiki.org" }, { "rank": 893, "domain": "investing.com", "state": "found", "status": 200, "robotsTxt": "https://www.investing.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/investing.com" }, { "rank": 894, "domain": "hrw.org", "state": "found", "status": 200, "robotsTxt": "https://hrw.org/robots.txt", "blocked": [ "AhrefsBot", "SemrushBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/hrw.org" }, { "rank": 895, "domain": "waze.com", "state": "found", "status": 200, "robotsTxt": "https://www.waze.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "AhrefsBot", "SemrushBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/waze.com" }, { "rank": 896, "domain": "last.fm", "state": "found", "status": 200, "robotsTxt": "https://www.last.fm/robots.txt", "blocked": [], "partlyBlocked": [ "Bingbot", "YandexBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/last.fm" }, { "rank": 897, "domain": "hyatt.com", "state": "refused", "status": 403, "robotsTxt": "https://www.hyatt.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/hyatt.com" }, { "rank": 898, "domain": "nyc.gov", "state": "found", "status": 200, "robotsTxt": "https://www.nyc.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nyc.gov" }, { "rank": 899, "domain": "prezi.com", "state": "found", "status": 200, "robotsTxt": "https://prezi.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [ "OAI-SearchBot", "Claude-SearchBot", "Claude-User" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/prezi.com" }, { "rank": 900, "domain": "gimp.org", "state": "found", "status": 200, "robotsTxt": "https://www.gimp.org/robots-www.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gimp.org" }, { "rank": 901, "domain": "panasonic.com", "state": "refused", "status": 403, "robotsTxt": "https://www.panasonic.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/panasonic.com" }, { "rank": 902, "domain": "heyzine.com", "state": "found", "status": 200, "robotsTxt": "https://heyzine.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/heyzine.com" }, { "rank": 904, "domain": "cisa.gov", "state": "found", "status": 200, "robotsTxt": "https://www.cisa.gov/robots.txt", "blocked": [ "PetalBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cisa.gov" }, { "rank": 905, "domain": "libsyn.com", "state": "found", "status": 200, "robotsTxt": "https://libsyn.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/libsyn.com" }, { "rank": 906, "domain": "osha.gov", "state": "refused", "status": 403, "robotsTxt": "https://osha.gov/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/osha.gov" }, { "rank": 907, "domain": "ncsu.edu", "state": "html", "status": 202, "robotsTxt": "https://www.ncsu.edu/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/ncsu.edu" }, { "rank": 908, "domain": "lego.com", "state": "found", "status": 200, "robotsTxt": "https://www.lego.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/lego.com" }, { "rank": 910, "domain": "isc.org", "state": "found", "status": 200, "robotsTxt": "https://www.isc.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/isc.org" }, { "rank": 911, "domain": "welt.de", "state": "found", "status": 200, "robotsTxt": "https://www.welt.de/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "PerplexityBot", "DuckAssistBot", "Amazonbot", "YouBot", "CCBot", "Applebot-Extended", "Bytespider", "Perplexity-User", "MistralAI-User", "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/welt.de" }, { "rank": 912, "domain": "navy.mil", "state": "refused", "status": 403, "robotsTxt": "https://www.navy.mil/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/navy.mil" }, { "rank": 913, "domain": "amap.com", "state": "found", "status": 200, "robotsTxt": "https://amap.com/robots.txt", "blocked": [ "DuckDuckBot", "PetalBot", "YandexBot", "Claude-SearchBot", "DuckAssistBot", "ClaudeBot", "Applebot-Extended", "GoogleOther", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Amazonbot", "YouBot", "GPTBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/amap.com" }, { "rank": 914, "domain": "sfu.ca", "state": "found", "status": 200, "robotsTxt": "https://www.sfu.ca/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/sfu.ca" }, { "rank": 915, "domain": "lifehacker.com", "state": "found", "status": 200, "robotsTxt": "https://lifehacker.com/robots.txt", "blocked": [ "Applebot", "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/lifehacker.com" }, { "rank": 917, "domain": "ucsb.edu", "state": "found", "status": 200, "robotsTxt": "https://www.ucsb.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ucsb.edu" }, { "rank": 918, "domain": "colorlib.com", "state": "found", "status": 200, "robotsTxt": "https://colorlib.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/colorlib.com" }, { "rank": 919, "domain": "vkontakte.ru", "state": "found", "status": 200, "robotsTxt": "https://vkontakte.ru/robots.txt", "blocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/vkontakte.ru" }, { "rank": 920, "domain": "nba.com", "state": "found", "status": 200, "robotsTxt": "https://www.nba.com/robots.txt", "blocked": [ "PerplexityBot", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nba.com" }, { "rank": 922, "domain": "tesla.com", "state": "refused", "status": 403, "robotsTxt": "https://tesla.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/tesla.com" }, { "rank": 923, "domain": "zimbra.com", "state": "found", "status": 200, "robotsTxt": "https://zimbra.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/zimbra.com" }, { "rank": 924, "domain": "theregister.com", "state": "found", "status": 200, "robotsTxt": "https://www.theregister.com/robots.txt", "blocked": [ "PetalBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/theregister.com" }, { "rank": 925, "domain": "caltech.edu", "state": "found", "status": 200, "robotsTxt": "https://www.caltech.edu/robots.txt", "blocked": [ "SemrushBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/caltech.edu" }, { "rank": 926, "domain": "zapier.com", "state": "found", "status": 200, "robotsTxt": "https://zapier.com/robots.txt", "blocked": [], "partlyBlocked": [ "Baiduspider" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/zapier.com" }, { "rank": 927, "domain": "domaintools.com", "state": "found", "status": 200, "robotsTxt": "https://www.domaintools.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/domaintools.com" }, { "rank": 928, "domain": "news.com.au", "state": "found", "status": 200, "robotsTxt": "https://www.news.com.au/robots.txt", "blocked": [ "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "GoogleOther" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/news.com.au" }, { "rank": 930, "domain": "mlb.com", "state": "refused", "status": 403, "robotsTxt": "https://mlb.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/mlb.com" }, { "rank": 931, "domain": "americanexpress.com", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/americanexpress.com" }, { "rank": 932, "domain": "hootsuite.com", "state": "found", "status": 200, "robotsTxt": "https://www.hootsuite.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "OAI-SearchBot", "PerplexityBot", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "Applebot-Extended", "Meta-ExternalAgent", "ChatGPT-User", "Claude-User", "Perplexity-User" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/hootsuite.com" }, { "rank": 933, "domain": "target.com", "state": "found", "status": 200, "robotsTxt": "https://www.target.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/target.com" }, { "rank": 934, "domain": "fool.com", "state": "found", "status": 200, "robotsTxt": "https://www.fool.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "CCBot", "Bytespider" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/fool.com" }, { "rank": 935, "domain": "weather.gov", "state": "none", "status": 404, "robotsTxt": "https://www.weather.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/weather.gov" }, { "rank": 936, "domain": "iqiyi.com", "state": "found", "status": 200, "robotsTxt": "http://www.iqiyi.com/robots.txt", "blocked": [ "PetalBot", "PerplexityBot", "GPTBot", "ClaudeBot", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/iqiyi.com" }, { "rank": 937, "domain": "ispconfig.org", "state": "found", "status": 200, "robotsTxt": "https://ispconfig.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ispconfig.org" }, { "rank": 938, "domain": "securityfocus.com", "state": "found", "status": 200, "robotsTxt": "https://securityfocus.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/securityfocus.com" }, { "rank": 939, "domain": "max.ru", "state": "found", "status": 200, "robotsTxt": "https://max.ru/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/max.ru" }, { "rank": 940, "domain": "wikibooks.org", "state": "found", "status": 200, "robotsTxt": "https://en.wikibooks.org/robots.txt", "blocked": [], "partlyBlocked": [ "SemrushBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/wikibooks.org" }, { "rank": 941, "domain": "tableau.com", "state": "refused", "status": 403, "robotsTxt": "https://tableau.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/tableau.com" }, { "rank": 943, "domain": "ec-lyon.fr", "state": "found", "status": 200, "robotsTxt": "https://www.ec-lyon.fr/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ec-lyon.fr" }, { "rank": 944, "domain": "nymag.com", "state": "found", "status": 200, "robotsTxt": "https://nymag.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nymag.com" }, { "rank": 945, "domain": "perl.com", "state": "found", "status": 200, "robotsTxt": "https://www.perl.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/perl.com" }, { "rank": 946, "domain": "narod.ru", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/narod.ru" }, { "rank": 947, "domain": "digitaltrends.com", "state": "found", "status": 200, "robotsTxt": "https://www.digitaltrends.com/robots.txt", "blocked": [ "PetalBot", "YandexBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Bytespider" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "Amazonbot", "YouBot", "Meta-ExternalAgent", "ChatGPT-User", "Claude-User", "Perplexity-User", "AhrefsBot", "SemrushBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/digitaltrends.com" }, { "rank": 948, "domain": "themegrill.com", "state": "error", "status": 520, "robotsTxt": "https://www.themegrill.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/themegrill.com" }, { "rank": 950, "domain": "amnesty.org", "state": "found", "status": 200, "robotsTxt": "https://www.amnesty.org/robots.txt", "blocked": [ "Bingbot", "PerplexityBot", "CCBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/amnesty.org" }, { "rank": 951, "domain": "ko-fi.com", "state": "found", "status": 200, "robotsTxt": "https://ko-fi.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ko-fi.com" }, { "rank": 952, "domain": "vogue.com", "state": "found", "status": 200, "robotsTxt": "https://www.vogue.com/robots.txt", "blocked": [ "PetalBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/vogue.com" }, { "rank": 954, "domain": "ourworldindata.org", "state": "found", "status": 200, "robotsTxt": "https://ourworldindata.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ourworldindata.org" }, { "rank": 956, "domain": "georgetown.edu", "state": "found", "status": 200, "robotsTxt": "https://www.georgetown.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/georgetown.edu" }, { "rank": 958, "domain": "consumerreports.org", "state": "found", "status": 200, "robotsTxt": "https://www.consumerreports.org/robots.txt", "blocked": [ "GPTBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/consumerreports.org" }, { "rank": 959, "domain": "tradingview.com", "state": "found", "status": 200, "robotsTxt": "https://www.tradingview.com/robots.txt", "blocked": [], "partlyBlocked": [ "Bingbot", "OAI-SearchBot", "PerplexityBot", "Amazonbot", "ClaudeBot", "Google-Extended", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/tradingview.com" }, { "rank": 960, "domain": "aepd.es", "state": "found", "status": 200, "robotsTxt": "https://www.aepd.es/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/aepd.es" }, { "rank": 961, "domain": "metmuseum.org", "state": "refused", "status": 429, "robotsTxt": "https://metmuseum.org/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/metmuseum.org" }, { "rank": 962, "domain": "yumpu.com", "state": "found", "status": 200, "robotsTxt": "https://yumpu.com/robots.txt", "blocked": [ "YandexBot", "PerplexityBot", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "Meta-ExternalFetcher", "AhrefsBot", "DotBot" ], "partlyBlocked": [ "Bingbot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/yumpu.com" }, { "rank": 963, "domain": "iheart.com", "state": "found", "status": 200, "robotsTxt": "https://www.iheart.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [ "Applebot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "Amazonbot", "ChatGPT-User", "Claude-User", "Perplexity-User", "Meta-ExternalFetcher" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/iheart.com" }, { "rank": 964, "domain": "searchengineland.com", "state": "refused", "status": 403, "robotsTxt": "https://searchengineland.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/searchengineland.com" }, { "rank": 965, "domain": "hopkinsmedicine.org", "state": "found", "status": 200, "robotsTxt": "https://www.hopkinsmedicine.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/hopkinsmedicine.org" }, { "rank": 967, "domain": "legislation.gov.uk", "state": "found", "status": 200, "robotsTxt": "https://www.legislation.gov.uk/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/legislation.gov.uk" }, { "rank": 968, "domain": "upwork.com", "state": "found", "status": 200, "robotsTxt": "https://www.upwork.com/robots.txt", "blocked": [ "PetalBot" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/upwork.com" }, { "rank": 969, "domain": "att.com", "state": "found", "status": 200, "robotsTxt": "https://www.att.com/robots.txt", "blocked": [ "DotBot" ], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/att.com" }, { "rank": 970, "domain": "metro.co.uk", "state": "found", "status": 200, "robotsTxt": "https://metro.co.uk/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/metro.co.uk" }, { "rank": 971, "domain": "nokia.com", "state": "refused", "status": 403, "robotsTxt": "https://www.nokia.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/nokia.com" }, { "rank": 972, "domain": "today.com", "state": "found", "status": 200, "robotsTxt": "https://www.today.com/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/today.com" }, { "rank": 973, "domain": "medscape.com", "state": "found", "status": 200, "robotsTxt": "https://www.medscape.com/robots.txt", "blocked": [ "PerplexityBot", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/medscape.com" }, { "rank": 974, "domain": "bell-labs.com", "state": "none", "status": 410, "robotsTxt": "https://www.bell-labs.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bell-labs.com" }, { "rank": 975, "domain": "justia.com", "state": "found", "status": 200, "robotsTxt": "https://www.justia.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/justia.com" }, { "rank": 976, "domain": "flaticon.com", "state": "found", "status": 200, "robotsTxt": "https://www.flaticon.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/flaticon.com" }, { "rank": 977, "domain": "clinicaltrials.gov", "state": "found", "status": 200, "robotsTxt": "https://clinicaltrials.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/clinicaltrials.gov" }, { "rank": 978, "domain": "xkcd.com", "state": "found", "status": 200, "robotsTxt": "https://xkcd.com/robots.txt", "blocked": [ "GPTBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/xkcd.com" }, { "rank": 979, "domain": "ning.com", "state": "found", "status": 200, "robotsTxt": "https://www.ning.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ning.com" }, { "rank": 980, "domain": "mcgill.ca", "state": "found", "status": 200, "robotsTxt": "https://mcgill.ca/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/mcgill.ca" }, { "rank": 982, "domain": "tomshardware.com", "state": "found", "status": 200, "robotsTxt": "https://www.tomshardware.com/robots.txt", "blocked": [ "Applebot", "Amazonbot", "YouBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/tomshardware.com" }, { "rank": 984, "domain": "xiaohongshu.com", "state": "found", "status": 200, "robotsTxt": "https://www.xiaohongshu.com/robots.txt", "blocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/xiaohongshu.com" }, { "rank": 985, "domain": "pitt.edu", "state": "found", "status": 200, "robotsTxt": "https://www.pitt.edu//robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/pitt.edu" }, { "rank": 986, "domain": "zeit.de", "state": "found", "status": 200, "robotsTxt": "https://www.zeit.de/robots.txt", "blocked": [ "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "PetalBot", "YandexBot", "GoogleOther" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/zeit.de" }, { "rank": 987, "domain": "codeberg.org", "state": "found", "status": 200, "robotsTxt": "https://codeberg.org/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/codeberg.org" }, { "rank": 988, "domain": "feedly.com", "state": "found", "status": 200, "robotsTxt": "https://feedly.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/feedly.com" }, { "rank": 989, "domain": "boston.com", "state": "found", "status": 200, "robotsTxt": "https://www.boston.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/boston.com" }, { "rank": 990, "domain": "salon.com", "state": "found", "status": 200, "robotsTxt": "https://www.salon.com/robots.txt", "blocked": [ "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/salon.com" }, { "rank": 991, "domain": "brandbucket.com", "state": "found", "status": 200, "robotsTxt": "https://www.brandbucket.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/brandbucket.com" }, { "rank": 992, "domain": "nydailynews.com", "state": "found", "status": 200, "robotsTxt": "https://www.nydailynews.com/robots.txt", "blocked": [ "Applebot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nydailynews.com" }, { "rank": 993, "domain": "cia.gov", "state": "found", "status": 200, "robotsTxt": "https://www.cia.gov/robots.txt", "blocked": [ "Bingbot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cia.gov" }, { "rank": 994, "domain": "mariadb.org", "state": "found", "status": 200, "robotsTxt": "https://mariadb.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/mariadb.org" }, { "rank": 995, "domain": "hbs.edu", "state": "found", "status": 200, "robotsTxt": "https://www.hbs.edu/robots.txt", "blocked": [], "partlyBlocked": [ "Bingbot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/hbs.edu" }, { "rank": 996, "domain": "chron.com", "state": "found", "status": 200, "robotsTxt": "https://www.chron.com/robots.txt", "blocked": [ "PerplexityBot", "DuckAssistBot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Bytespider", "Claude-User", "Perplexity-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/chron.com" }, { "rank": 997, "domain": "trendmicro.com", "state": "found", "status": 200, "robotsTxt": "https://www.trendmicro.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "ClaudeBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/trendmicro.com" }, { "rank": 998, "domain": "eurekalert.org", "state": "found", "status": 200, "robotsTxt": "https://www.eurekalert.org/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "Bytespider", "AhrefsBot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/eurekalert.org" }, { "rank": 999, "domain": "istockphoto.com", "state": "found", "status": 200, "robotsTxt": "https://www.istockphoto.com/robots.txt", "blocked": [ "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/istockphoto.com" }, { "rank": 1000, "domain": "sueddeutsche.de", "state": "found", "status": 200, "robotsTxt": "https://www.sueddeutsche.de/robots.txt", "blocked": [ "PerplexityBot", "GPTBot", "ClaudeBot", "Google-Extended", "ChatGPT-User", "AhrefsBot", "SemrushBot" ], "partlyBlocked": [ "Amazonbot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/sueddeutsche.de" }, { "rank": 1001, "domain": "ndtv.com", "state": "refused", "status": 403, "robotsTxt": "https://www.ndtv.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/ndtv.com" }, { "rank": 1002, "domain": "federalreserve.gov", "state": "none", "status": 404, "robotsTxt": "https://www.federalreserve.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/federalreserve.gov" }, { "rank": 1004, "domain": "zillow.com", "state": "found", "status": 200, "robotsTxt": "https://www.zillow.com/robots.txt", "blocked": [ "Amazonbot", "YouBot", "Applebot-Extended", "Meta-ExternalAgent" ], "partlyBlocked": [ "Claude-SearchBot", "ClaudeBot", "Claude-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/zillow.com" }, { "rank": 1005, "domain": "routledge.com", "state": "found", "status": 200, "robotsTxt": "https://www.routledge.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/routledge.com" }, { "rank": 1006, "domain": "padlet.com", "state": "found", "status": 200, "robotsTxt": "https://padlet.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/padlet.com" }, { "rank": 1007, "domain": "mhlw.go.jp", "state": "found", "status": 200, "robotsTxt": "https://www.mhlw.go.jp/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/mhlw.go.jp" }, { "rank": 1008, "domain": "fedex.com", "state": "found", "status": 200, "robotsTxt": "https://www.fedex.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/fedex.com" }, { "rank": 1011, "domain": "thedailybeast.com", "state": "found", "status": 200, "robotsTxt": "https://www.thedailybeast.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/thedailybeast.com" }, { "rank": 1012, "domain": "lonelyplanet.com", "state": "found", "status": 200, "robotsTxt": "https://www.lonelyplanet.com/robots.txt", "blocked": [ "GPTBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/lonelyplanet.com" }, { "rank": 1013, "domain": "foodnetwork.com", "state": "found", "status": 200, "robotsTxt": "https://www.foodnetwork.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/foodnetwork.com" }, { "rank": 1014, "domain": "rsc.org", "state": "found", "status": 200, "robotsTxt": "https://www.rsc.org/robots.txt", "blocked": [ "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/rsc.org" }, { "rank": 1015, "domain": "pcworld.com", "state": "found", "status": 200, "robotsTxt": "https://www.pcworld.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/pcworld.com" }, { "rank": 1016, "domain": "pastebin.com", "state": "found", "status": 200, "robotsTxt": "https://pastebin.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/pastebin.com" }, { "rank": 1017, "domain": "videolan.org", "state": "found", "status": 200, "robotsTxt": "https://www.videolan.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/videolan.org" }, { "rank": 1018, "domain": "verizon.com", "state": "found", "status": 200, "robotsTxt": "https://verizon.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/verizon.com" }, { "rank": 1020, "domain": "lse.ac.uk", "state": "found", "status": 200, "robotsTxt": "https://www.lse.ac.uk/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/lse.ac.uk" }, { "rank": 1021, "domain": "urbandictionary.com", "state": "found", "status": 200, "robotsTxt": "https://www.urbandictionary.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/urbandictionary.com" }, { "rank": 1023, "domain": "hindustantimes.com", "state": "found", "status": 200, "robotsTxt": "https://www.hindustantimes.com/robots.txt", "blocked": [ "Meta-ExternalAgent" ], "partlyBlocked": [ "Googlebot", "Applebot", "Google-Extended" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/hindustantimes.com" }, { "rank": 1024, "domain": "dzen.ru", "state": "found", "status": 200, "robotsTxt": "https://dzen.ru/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/dzen.ru" }, { "rank": 1025, "domain": "dev.to", "state": "found", "status": 200, "robotsTxt": "https://dev.to/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/dev.to" }, { "rank": 1026, "domain": "500px.com", "state": "found", "status": 200, "robotsTxt": "https://500px.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/500px.com" }, { "rank": 1027, "domain": "sba.gov", "state": "found", "status": 200, "robotsTxt": "https://www.sba.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/sba.gov" }, { "rank": 1028, "domain": "zenodo.org", "state": "found", "status": 200, "robotsTxt": "https://zenodo.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/zenodo.org" }, { "rank": 1029, "domain": "thefreedictionary.com", "state": "found", "status": 200, "robotsTxt": "https://www.thefreedictionary.com/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/thefreedictionary.com" }, { "rank": 1030, "domain": "brown.edu", "state": "found", "status": 200, "robotsTxt": "https://www.brown.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/brown.edu" }, { "rank": 1031, "domain": "mega.nz", "state": "found", "status": 200, "robotsTxt": "https://mega.nz/robots.txt", "blocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/mega.nz" }, { "rank": 1032, "domain": "lg.com", "state": "found", "status": 200, "robotsTxt": "https://www.lg.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/lg.com" }, { "rank": 1033, "domain": "thenextweb.com", "state": "found", "status": 200, "robotsTxt": "https://thenextweb.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/thenextweb.com" }, { "rank": 1034, "domain": "tufts.edu", "state": "found", "status": 200, "robotsTxt": "https://www.tufts.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/tufts.edu" }, { "rank": 1035, "domain": "houzz.com", "state": "found", "status": 200, "robotsTxt": "https://www.houzz.com/robots.txt", "blocked": [], "partlyBlocked": [ "OAI-SearchBot", "PerplexityBot", "GPTBot", "ClaudeBot", "Google-Extended", "ChatGPT-User", "Claude-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/houzz.com" }, { "rank": 1036, "domain": "usps.com", "state": "found", "status": 200, "robotsTxt": "https://www.usps.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/usps.com" }, { "rank": 1037, "domain": "gamespot.com", "state": "found", "status": 200, "robotsTxt": "https://gamespot.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gamespot.com" }, { "rank": 1038, "domain": "hawaii.edu", "state": "found", "status": 200, "robotsTxt": "https://hawaii.edu/robots.txt", "blocked": [], "partlyBlocked": [ "Baiduspider" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/hawaii.edu" }, { "rank": 1039, "domain": "wunderground.com", "state": "found", "status": 200, "robotsTxt": "https://www.wunderground.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/wunderground.com" }, { "rank": 1040, "domain": "sony.com", "state": "refused", "status": 403, "robotsTxt": "https://www.sony.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/sony.com" }, { "rank": 1042, "domain": "hpe.com", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/hpe.com" }, { "rank": 1044, "domain": "squareup.com", "state": "found", "status": 200, "robotsTxt": "https://squareup.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/squareup.com" }, { "rank": 1046, "domain": "prestashop.com", "state": "found", "status": 200, "robotsTxt": "https://prestashop.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/prestashop.com" }, { "rank": 1047, "domain": "deepseek.com", "state": "found", "status": 200, "robotsTxt": "https://deepseek.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/deepseek.com" }, { "rank": 1048, "domain": "space.com", "state": "found", "status": 200, "robotsTxt": "https://www.space.com/robots.txt", "blocked": [ "Applebot", "Amazonbot", "YouBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/space.com" }, { "rank": 1050, "domain": "ups.com", "state": "html", "status": 200, "robotsTxt": "https://www.ups.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/ups.com" }, { "rank": 1051, "domain": "360.cn", "state": "found", "status": 200, "robotsTxt": "https://360.cn/robots.txt", "blocked": [ "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/360.cn" }, { "rank": 1053, "domain": "zlib.net", "state": "found", "status": 200, "robotsTxt": "https://zlib.net/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/zlib.net" }, { "rank": 1054, "domain": "discogs.com", "state": "refused", "status": 403, "robotsTxt": "https://www.discogs.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/discogs.com" }, { "rank": 1055, "domain": "bitly.com", "state": "found", "status": 200, "robotsTxt": "https://bitly.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bitly.com" }, { "rank": 1056, "domain": "deadline.com", "state": "found", "status": 200, "robotsTxt": "https://deadline.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/deadline.com" }, { "rank": 1057, "domain": "cwi.nl", "state": "found", "status": 200, "robotsTxt": "https://www.cwi.nl/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cwi.nl" }, { "rank": 1058, "domain": "gatech.edu", "state": "found", "status": 200, "robotsTxt": "https://www.gatech.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gatech.edu" }, { "rank": 1059, "domain": "ntp.org", "state": "found", "status": 200, "robotsTxt": "https://www.ntp.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ntp.org" }, { "rank": 1060, "domain": "apachelounge.com", "state": "found", "status": 200, "robotsTxt": "https://www.apachelounge.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/apachelounge.com" }, { "rank": 1061, "domain": "squid-cache.org", "state": "found", "status": 200, "robotsTxt": "https://www.squid-cache.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/squid-cache.org" }, { "rank": 1062, "domain": "billboard.com", "state": "found", "status": 200, "robotsTxt": "https://www.billboard.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/billboard.com" }, { "rank": 1063, "domain": "voanews.com", "state": "found", "status": 200, "robotsTxt": "https://www.voanews.com/robots.txt", "blocked": [ "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/voanews.com" }, { "rank": 1064, "domain": "vt.edu", "state": "found", "status": 200, "robotsTxt": "https://www.vt.edu/content/dam/vt_edu/robots.txt", "blocked": [], "partlyBlocked": [ "Bingbot", "SemrushBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/vt.edu" }, { "rank": 1066, "domain": "ssa.gov", "state": "refused", "status": 403, "robotsTxt": "https://www.ssa.gov/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/ssa.gov" }, { "rank": 1067, "domain": "airtable.com", "state": "found", "status": 200, "robotsTxt": "https://airtable.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/airtable.com" }, { "rank": 1069, "domain": "grandviewresearch.com", "state": "found", "status": 200, "robotsTxt": "https://www.grandviewresearch.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/grandviewresearch.com" }, { "rank": 1070, "domain": "dhl.com", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/dhl.com" }, { "rank": 1071, "domain": "dictionary.com", "state": "found", "status": 200, "robotsTxt": "https://www.dictionary.com/robots.txt", "blocked": [ "GPTBot", "CCBot", "Applebot-Extended" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/dictionary.com" }, { "rank": 1072, "domain": "over-blog.com", "state": "found", "status": 200, "robotsTxt": "https://www.over-blog.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/over-blog.com" }, { "rank": 1073, "domain": "epicgames.com", "state": "refused", "status": 403, "robotsTxt": "https://www.epicgames.com/site/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/epicgames.com" }, { "rank": 1074, "domain": "computerworld.com", "state": "found", "status": 200, "robotsTxt": "https://www.computerworld.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/computerworld.com" }, { "rank": 1075, "domain": "unity.com", "state": "found", "status": 200, "robotsTxt": "https://unity.com/robots.txt", "blocked": [ "Baiduspider", "Bytespider" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/unity.com" }, { "rank": 1077, "domain": "gitea.com", "state": "found", "status": 200, "robotsTxt": "https://gitea.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "GPTBot", "ClaudeBot", "CCBot", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gitea.com" }, { "rank": 1078, "domain": "faz.net", "state": "found", "status": 200, "robotsTxt": "https://www.faz.net/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "MistralAI-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/faz.net" }, { "rank": 1079, "domain": "spglobal.com", "state": "refused", "status": 403, "robotsTxt": "https://www.spglobal.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/spglobal.com" }, { "rank": 1080, "domain": "sketchfab.com", "state": "found", "status": 200, "robotsTxt": "https://sketchfab.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/sketchfab.com" }, { "rank": 1083, "domain": "unl.edu", "state": "found", "status": 200, "robotsTxt": "https://www.unl.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/unl.edu" }, { "rank": 1085, "domain": "jpost.com", "state": "found", "status": 200, "robotsTxt": "https://www.jpost.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot", "OAI-SearchBot", "GPTBot", "ChatGPT-User", "Claude-User", "Perplexity-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/jpost.com" }, { "rank": 1086, "domain": "bostonglobe.com", "state": "found", "status": 200, "robotsTxt": "https://www.bostonglobe.com/robots.txt", "blocked": [ "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bostonglobe.com" }, { "rank": 1087, "domain": "unfccc.int", "state": "found", "status": 200, "robotsTxt": "https://unfccc.int/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/unfccc.int" }, { "rank": 1088, "domain": "aarp.org", "state": "found", "status": 200, "robotsTxt": "https://www.aarp.org/robots.txt", "blocked": [ "Baiduspider", "CCBot", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/aarp.org" }, { "rank": 1091, "domain": "edx.org", "state": "found", "status": 200, "robotsTxt": "https://www.edx.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/edx.org" }, { "rank": 1092, "domain": "nber.org", "state": "found", "status": 200, "robotsTxt": "https://www.nber.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nber.org" }, { "rank": 1093, "domain": "thestar.com", "state": "found", "status": 200, "robotsTxt": "https://thestar.com/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/thestar.com" }, { "rank": 1094, "domain": "one.com", "state": "found", "status": 200, "robotsTxt": "https://www.one.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/one.com" }, { "rank": 1095, "domain": "vanityfair.com", "state": "found", "status": 200, "robotsTxt": "https://www.vanityfair.com/robots.txt", "blocked": [ "PetalBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/vanityfair.com" }, { "rank": 1097, "domain": "fraunhofer.de", "state": "found", "status": 200, "robotsTxt": "https://www.fraunhofer.de/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/fraunhofer.de" }, { "rank": 1098, "domain": "sophos.com", "state": "found", "status": 200, "robotsTxt": "https://www.sophos.com/robots.txt", "blocked": [ "PetalBot", "Amazonbot" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/sophos.com" }, { "rank": 1100, "domain": "irishtimes.com", "state": "found", "status": 200, "robotsTxt": "https://www.irishtimes.com/robots.txt", "blocked": [ "GPTBot", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/irishtimes.com" }, { "rank": 1103, "domain": "worldcat.org", "state": "found", "status": 200, "robotsTxt": "https://search.worldcat.org/robots.txt", "blocked": [ "GPTBot", "Google-Extended", "CCBot", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/worldcat.org" }, { "rank": 1104, "domain": "timeout.com", "state": "found", "status": 200, "robotsTxt": "https://www.timeout.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/timeout.com" }, { "rank": 1105, "domain": "repubblica.it", "state": "found", "status": 200, "robotsTxt": "https://www.repubblica.it/robots.txt", "blocked": [ "Applebot", "PetalBot", "PerplexityBot", "DuckAssistBot", "YouBot", "GPTBot", "Google-Extended", "CCBot", "Applebot-Extended", "Bytespider", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/repubblica.it" }, { "rank": 1106, "domain": "boutell.com", "state": "found", "status": 200, "robotsTxt": "https://boutell.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/boutell.com" }, { "rank": 1107, "domain": "thawte.com", "state": "found", "status": 200, "robotsTxt": "https://www.thawte.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/thawte.com" }, { "rank": 1108, "domain": "site.com", "state": "found", "status": 200, "robotsTxt": "https://site.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/site.com" }, { "rank": 1109, "domain": "asana.com", "state": "found", "status": 200, "robotsTxt": "https://asana.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/asana.com" }, { "rank": 1111, "domain": "ghost.org", "state": "found", "status": 200, "robotsTxt": "https://ghost.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ghost.org" }, { "rank": 1113, "domain": "bluehost.com", "state": "refused", "status": 403, "robotsTxt": "https://www.bluehost.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/bluehost.com" }, { "rank": 1115, "domain": "strava.com", "state": "found", "status": 200, "robotsTxt": "https://www.strava.com/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "Google-Extended", "Meta-ExternalAgent" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/strava.com" }, { "rank": 1116, "domain": "eia.gov", "state": "found", "status": 200, "robotsTxt": "https://www.eia.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/eia.gov" }, { "rank": 1117, "domain": "asahi.com", "state": "found", "status": 200, "robotsTxt": "https://www.asahi.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "Meta-ExternalFetcher" ], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/asahi.com" }, { "rank": 1118, "domain": "libpng.org", "state": "none", "status": 404, "robotsTxt": "https://libpng.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/libpng.org" }, { "rank": 1120, "domain": "penguinrandomhouse.com", "state": "found", "status": 200, "robotsTxt": "https://www.penguinrandomhouse.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/penguinrandomhouse.com" }, { "rank": 1122, "domain": "grammarly.com", "state": "found", "status": 200, "robotsTxt": "https://www.grammarly.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/grammarly.com" }, { "rank": 1123, "domain": "archlinux.org", "state": "found", "status": 200, "robotsTxt": "https://archlinux.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/archlinux.org" }, { "rank": 1124, "domain": "foursquare.com", "state": "found", "status": 200, "robotsTxt": "https://foursquare.com/robots.txt", "blocked": [ "GPTBot", "ChatGPT-User" ], "partlyBlocked": [ "CCBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/foursquare.com" }, { "rank": 1125, "domain": "phpbb.com", "state": "refused", "status": 403, "robotsTxt": "https://phpbb.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/phpbb.com" }, { "rank": 1126, "domain": "indianexpress.com", "state": "refused", "status": 403, "robotsTxt": "https://indianexpress.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/indianexpress.com" }, { "rank": 1128, "domain": "hotjar.com", "state": "found", "status": 200, "robotsTxt": "https://hotjar.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/hotjar.com" }, { "rank": 1129, "domain": "note.com", "state": "found", "status": 200, "robotsTxt": "https://note.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/note.com" }, { "rank": 1131, "domain": "fema.gov", "state": "found", "status": 200, "robotsTxt": "https://www.fema.gov/robots.txt", "blocked": [ "PetalBot", "Meta-ExternalAgent" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/fema.gov" }, { "rank": 1132, "domain": "thomsonreuters.com", "state": "found", "status": 200, "robotsTxt": "https://www.thomsonreuters.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/thomsonreuters.com" }, { "rank": 1133, "domain": "readthedocs.org", "state": "found", "status": 200, "robotsTxt": "https://app.readthedocs.org/robots.txt", "blocked": [ "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/readthedocs.org" }, { "rank": 1134, "domain": "gohugo.io", "state": "none", "status": 404, "robotsTxt": "https://gohugo.io/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gohugo.io" }, { "rank": 1136, "domain": "wampserver.com", "state": "none", "status": 410, "robotsTxt": "https://www.wampserver.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/wampserver.com" }, { "rank": 1137, "domain": "japantimes.co.jp", "state": "found", "status": 200, "robotsTxt": "https://www.japantimes.co.jp/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/japantimes.co.jp" }, { "rank": 1138, "domain": "gitee.com", "state": "found", "status": 200, "robotsTxt": "https://gitee.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/gitee.com" }, { "rank": 1139, "domain": "thehindu.com", "state": "found", "status": 200, "robotsTxt": "https://www.thehindu.com/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/thehindu.com" }, { "rank": 1141, "domain": "unhcr.org", "state": "found", "status": 200, "robotsTxt": "https://www.unhcr.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/unhcr.org" }, { "rank": 1143, "domain": "gnome.org", "state": "none", "status": 404, "robotsTxt": "https://www.gnome.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gnome.org" }, { "rank": 1145, "domain": "uga.edu", "state": "found", "status": 200, "robotsTxt": "https://www.uga.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/uga.edu" }, { "rank": 1146, "domain": "t-online.de", "state": "found", "status": 200, "robotsTxt": "https://www.t-online.de/robots.txt", "blocked": [ "PerplexityBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [ "Google-Extended" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/t-online.de" }, { "rank": 1147, "domain": "memcached.org", "state": "found", "status": 200, "robotsTxt": "https://memcached.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/memcached.org" }, { "rank": 1148, "domain": "nationalacademies.org", "state": "found", "status": 200, "robotsTxt": "https://www.nationalacademies.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nationalacademies.org" }, { "rank": 1149, "domain": "buzzsprout.com", "state": "found", "status": 200, "robotsTxt": "https://www.buzzsprout.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/buzzsprout.com" }, { "rank": 1150, "domain": "accuweather.com", "state": "refused", "status": 403, "robotsTxt": "https://accuweather.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/accuweather.com" }, { "rank": 1152, "domain": "timesofisrael.com", "state": "found", "status": 200, "robotsTxt": "https://www.timesofisrael.com/robots.txt", "blocked": [ "Bingbot", "AhrefsBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/timesofisrael.com" }, { "rank": 1153, "domain": "wetransfer.com", "state": "found", "status": 200, "robotsTxt": "https://wetransfer.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/wetransfer.com" }, { "rank": 1154, "domain": "mapquest.com", "state": "found", "status": 200, "robotsTxt": "https://www.mapquest.com/robots.txt", "blocked": [ "DuckDuckBot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/mapquest.com" }, { "rank": 1156, "domain": "uwaterloo.ca", "state": "none", "status": 404, "robotsTxt": "https://uwaterloo.ca/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/uwaterloo.ca" }, { "rank": 1157, "domain": "glassdoor.com", "state": "found", "status": 200, "robotsTxt": "https://www.glassdoor.com/robots.txt", "blocked": [ "Baiduspider", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/glassdoor.com" }, { "rank": 1158, "domain": "pcre.org", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/pcre.org" }, { "rank": 1159, "domain": "globalnews.ca", "state": "found", "status": 200, "robotsTxt": "https://globalnews.ca/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/globalnews.ca" }, { "rank": 1160, "domain": "umass.edu", "state": "found", "status": 200, "robotsTxt": "https://www.umass.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/umass.edu" }, { "rank": 1161, "domain": "smugmug.com", "state": "found", "status": 200, "robotsTxt": "https://www.smugmug.com/robots.txt", "blocked": [ "DuckDuckBot", "PetalBot", "YandexBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "Baiduspider" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/smugmug.com" }, { "rank": 1162, "domain": "duolingo.com", "state": "found", "status": 200, "robotsTxt": "https://www.duolingo.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/duolingo.com" }, { "rank": 1163, "domain": "jst.go.jp", "state": "found", "status": 200, "robotsTxt": "https://www.jst.go.jp/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/jst.go.jp" }, { "rank": 1164, "domain": "arduino.cc", "state": "found", "status": 200, "robotsTxt": "https://www.arduino.cc/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/arduino.cc" }, { "rank": 1165, "domain": "oregonstate.edu", "state": "found", "status": 200, "robotsTxt": "https://oregonstate.edu/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/oregonstate.edu" }, { "rank": 1166, "domain": "directadmin.com", "state": "found", "status": 200, "robotsTxt": "https://directadmin.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/directadmin.com" }, { "rank": 1168, "domain": "redbull.com", "state": "found", "status": 200, "robotsTxt": "https://www.redbull.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/redbull.com" }, { "rank": 1170, "domain": "bcg.com", "state": "found", "status": 200, "robotsTxt": "https://www.bcg.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bcg.com" }, { "rank": 1171, "domain": "wufoo.com", "state": "found", "status": 200, "robotsTxt": "https://www.wufoo.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/wufoo.com" }, { "rank": 1172, "domain": "snopes.com", "state": "found", "status": 200, "robotsTxt": "https://www.snopes.com/robots.txt", "blocked": [ "Applebot", "PerplexityBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/snopes.com" }, { "rank": 1174, "domain": "rochester.edu", "state": "found", "status": 200, "robotsTxt": "https://www.rochester.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/rochester.edu" }, { "rank": 1176, "domain": "klarna.com", "state": "html", "status": 202, "robotsTxt": "https://www.klarna.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/klarna.com" }, { "rank": 1177, "domain": "coindesk.com", "state": "found", "status": 200, "robotsTxt": "https://www.coindesk.com/robots.txt", "blocked": [ "PerplexityBot", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/coindesk.com" }, { "rank": 1178, "domain": "notion.com", "state": "found", "status": 200, "robotsTxt": "https://www.notion.com/robots.txt", "blocked": [ "Amazonbot", "Meta-ExternalAgent", "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/notion.com" }, { "rank": 1180, "domain": "buymeacoffee.com", "state": "found", "status": 200, "robotsTxt": "https://buymeacoffee.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/buymeacoffee.com" }, { "rank": 1181, "domain": "findlaw.com", "state": "refused", "status": 403, "robotsTxt": "https://findlaw.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/findlaw.com" }, { "rank": 1182, "domain": "gzip.org", "state": "found", "status": 200, "robotsTxt": "https://gzip.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gzip.org" }, { "rank": 1184, "domain": "boe.es", "state": "found", "status": 200, "robotsTxt": "https://boe.es/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/boe.es" }, { "rank": 1185, "domain": "elmundo.es", "state": "found", "status": 200, "robotsTxt": "https://www.elmundo.es/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/elmundo.es" }, { "rank": 1187, "domain": "deepl.com", "state": "found", "status": 200, "robotsTxt": "https://www.deepl.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/deepl.com" }, { "rank": 1188, "domain": "logitech.com", "state": "found", "status": 200, "robotsTxt": "https://www.logitech.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/logitech.com" }, { "rank": 1192, "domain": "kadencewp.com", "state": "found", "status": 200, "robotsTxt": "https://www.kadencewp.com/robots.txt", "blocked": [ "PetalBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/kadencewp.com" }, { "rank": 1193, "domain": "searchenginejournal.com", "state": "found", "status": 200, "robotsTxt": "https://www.searchenginejournal.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/searchenginejournal.com" }, { "rank": 1194, "domain": "proquest.com", "state": "found", "status": 200, "robotsTxt": "https://www.proquest.com/robots.txt", "blocked": [ "GPTBot", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/proquest.com" }, { "rank": 1195, "domain": "seattletimes.com", "state": "html", "status": 202, "robotsTxt": "https://seattletimes.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/seattletimes.com" }, { "rank": 1196, "domain": "britishcouncil.org", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/britishcouncil.org" }, { "rank": 1197, "domain": "brevo.com", "state": "none", "status": 404, "robotsTxt": "https://brevo.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/brevo.com" }, { "rank": 1199, "domain": "freetype.org", "state": "found", "status": 200, "robotsTxt": "https://freetype.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/freetype.org" }, { "rank": 1200, "domain": "rbc.ru", "state": "found", "status": 200, "robotsTxt": "https://www.rbc.ru/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/rbc.ru" }, { "rank": 1201, "domain": "openoffice.org", "state": "found", "status": 200, "robotsTxt": "https://www.openoffice.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/openoffice.org" }, { "rank": 1202, "domain": "biblegateway.com", "state": "found", "status": 200, "robotsTxt": "https://www.biblegateway.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/biblegateway.com" }, { "rank": 1203, "domain": "ria.ru", "state": "found", "status": 200, "robotsTxt": "https://ria.ru/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ria.ru" }, { "rank": 1204, "domain": "namebright.com", "state": "found", "status": 200, "robotsTxt": "https://www.namebright.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/namebright.com" }, { "rank": 1205, "domain": "faa.gov", "state": "refused", "status": 403, "robotsTxt": "http://www.faa.gov/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/faa.gov" }, { "rank": 1206, "domain": "cuny.edu", "state": "found", "status": 200, "robotsTxt": "https://cuny.edu/robots.txt", "blocked": [ "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/cuny.edu" }, { "rank": 1207, "domain": "discovery.com", "state": "found", "status": 200, "robotsTxt": "https://www.discovery.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/discovery.com" }, { "rank": 1208, "domain": "sqlite.org", "state": "found", "status": 200, "robotsTxt": "https://sqlite.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/sqlite.org" }, { "rank": 1209, "domain": "cern.ch", "state": "none", "status": 404, "robotsTxt": "https://cern.ch/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cern.ch" }, { "rank": 1210, "domain": "rottentomatoes.com", "state": "found", "status": 200, "robotsTxt": "https://www.rottentomatoes.com/robots.txt", "blocked": [], "partlyBlocked": [ "Google-Extended" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/rottentomatoes.com" }, { "rank": 1212, "domain": "uefa.com", "state": "found", "status": 200, "robotsTxt": "https://www.uefa.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/uefa.com" }, { "rank": 1213, "domain": "unam.mx", "state": "found", "status": 200, "robotsTxt": "https://www.unam.mx/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/unam.mx" }, { "rank": 1214, "domain": "virustotal.com", "state": "found", "status": 200, "robotsTxt": "https://virustotal.com/robots.txt", "blocked": [], "partlyBlocked": [ "Baiduspider" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/virustotal.com" }, { "rank": 1215, "domain": "iastate.edu", "state": "found", "status": 200, "robotsTxt": "https://www.iastate.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/iastate.edu" }, { "rank": 1216, "domain": "raspberrypi.org", "state": "found", "status": 200, "robotsTxt": "https://www.raspberrypi.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/raspberrypi.org" }, { "rank": 1217, "domain": "ctvnews.ca", "state": "found", "status": 200, "robotsTxt": "https://www.ctvnews.ca/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "GPTBot", "CCBot", "ChatGPT-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ctvnews.ca" }, { "rank": 1218, "domain": "habr.com", "state": "found", "status": 200, "robotsTxt": "https://habr.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/habr.com" }, { "rank": 1219, "domain": "china.com.cn", "state": "found", "status": 200, "robotsTxt": "http://www.china.com.cn/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/china.com.cn" }, { "rank": 1220, "domain": "oxfordjournals.org", "state": "found", "status": 200, "robotsTxt": "https://www.oxfordjournals.org/robots.txt", "blocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/oxfordjournals.org" }, { "rank": 1221, "domain": "chromium.org", "state": "found", "status": 200, "robotsTxt": "https://www.chromium.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/chromium.org" }, { "rank": 1222, "domain": "codepen.io", "state": "found", "status": 200, "robotsTxt": "https://www.codepen.io/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/codepen.io" }, { "rank": 1223, "domain": "disney.com", "state": "found", "status": 200, "robotsTxt": "https://www.disney.com/robots.txt", "blocked": [ "GPTBot", "Google-Extended", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/disney.com" }, { "rank": 1224, "domain": "redcross.org", "state": "refused", "status": 403, "robotsTxt": "https://www.redcross.org/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/redcross.org" }, { "rank": 1226, "domain": "answers.com", "state": "found", "status": 200, "robotsTxt": "https://www.answers.com/robots.txt", "blocked": [], "partlyBlocked": [ "Googlebot", "Bingbot", "Applebot", "Baiduspider" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/answers.com" }, { "rank": 1227, "domain": "unicode.org", "state": "found", "status": 200, "robotsTxt": "https://unicode.org/robots.txt", "blocked": [ "Bingbot", "Baiduspider", "CCBot", "AhrefsBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/unicode.org" }, { "rank": 1229, "domain": "buffer.com", "state": "found", "status": 200, "robotsTxt": "https://buffer.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/buffer.com" }, { "rank": 1230, "domain": "blender.org", "state": "found", "status": 200, "robotsTxt": "https://blender.org/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/blender.org" }, { "rank": 1231, "domain": "macrumors.com", "state": "found", "status": 200, "robotsTxt": "https://www.macrumors.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/macrumors.com" }, { "rank": 1232, "domain": "modsecurity.org", "state": "found", "status": 200, "robotsTxt": "https://modsecurity.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/modsecurity.org" }, { "rank": 1233, "domain": "hardened-php.net", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/hardened-php.net" }, { "rank": 1235, "domain": "enlightenment.org", "state": "found", "status": 200, "robotsTxt": "https://www.enlightenment.org/robots.txt", "blocked": [ "PetalBot", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/enlightenment.org" }, { "rank": 1236, "domain": "cnrs.fr", "state": "found", "status": 200, "robotsTxt": "https://www.cnrs.fr/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cnrs.fr" }, { "rank": 1237, "domain": "defense.gov", "state": "refused", "status": 403, "robotsTxt": "https://defense.gov/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/defense.gov" }, { "rank": 1238, "domain": "uiowa.edu", "state": "found", "status": 200, "robotsTxt": "https://uiowa.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/uiowa.edu" }, { "rank": 1240, "domain": "bild.de", "state": "found", "status": 200, "robotsTxt": "https://www.bild.de/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "PerplexityBot", "DuckAssistBot", "Amazonbot", "YouBot", "CCBot", "Applebot-Extended", "Bytespider", "Perplexity-User", "MistralAI-User", "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bild.de" }, { "rank": 1241, "domain": "filezilla-project.org", "state": "found", "status": 200, "robotsTxt": "https://filezilla-project.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/filezilla-project.org" }, { "rank": 1243, "domain": "eclipse.org", "state": "found", "status": 200, "robotsTxt": "https://www.eclipse.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/eclipse.org" }, { "rank": 1244, "domain": "standard.co.uk", "state": "found", "status": 200, "robotsTxt": "https://www.standard.co.uk/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/standard.co.uk" }, { "rank": 1245, "domain": "gettyimages.com", "state": "found", "status": 200, "robotsTxt": "https://www.gettyimages.com/robots.txt", "blocked": [ "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gettyimages.com" }, { "rank": 1246, "domain": "corriere.it", "state": "found", "status": 200, "robotsTxt": "https://www.corriere.it/robots.txt", "blocked": [ "Applebot", "PetalBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/corriere.it" }, { "rank": 1247, "domain": "philips.com", "state": "found", "status": 200, "robotsTxt": "https://www.philips.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/philips.com" }, { "rank": 1248, "domain": "foreignpolicy.com", "state": "found", "status": 200, "robotsTxt": "https://foreignpolicy.com/robots.txt", "blocked": [ "PetalBot", "PerplexityBot", "DuckAssistBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/foreignpolicy.com" }, { "rank": 1250, "domain": "kakao.com", "state": "found", "status": 200, "robotsTxt": "https://kakao.com/robots.txt", "blocked": [ "Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "PetalBot", "YandexBot", "Baiduspider", "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/kakao.com" }, { "rank": 1254, "domain": "thebalancemoney.com", "state": "found", "status": 200, "robotsTxt": "https://www.thebalancemoney.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [ "OAI-SearchBot", "GPTBot", "ChatGPT-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/thebalancemoney.com" }, { "rank": 1256, "domain": "af.mil", "state": "refused", "status": 403, "robotsTxt": "https://www.af.mil/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/af.mil" }, { "rank": 1257, "domain": "uscourts.gov", "state": "found", "status": 200, "robotsTxt": "https://www.uscourts.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/uscourts.gov" }, { "rank": 1258, "domain": "livechat.com", "state": "found", "status": 200, "robotsTxt": "https://www.livechat.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/livechat.com" }, { "rank": 1259, "domain": "ahajournals.org", "state": "found", "status": 200, "robotsTxt": "https://www.ahajournals.org/robots.txt", "blocked": [ "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ahajournals.org" }, { "rank": 1261, "domain": "nsf.gov", "state": "found", "status": 200, "robotsTxt": "https://www.nsf.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nsf.gov" }, { "rank": 1262, "domain": "biglobe.ne.jp", "state": "none", "status": 404, "robotsTxt": "https://biglobe.ne.jp/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/biglobe.ne.jp" }, { "rank": 1263, "domain": "se.com", "state": "found", "status": 200, "robotsTxt": "https://www.se.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/se.com" }, { "rank": 1265, "domain": "howtogeek.com", "state": "error", "status": null, "robotsTxt": null, "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/howtogeek.com" }, { "rank": 1266, "domain": "govinfo.gov", "state": "found", "status": 200, "robotsTxt": "https://www.govinfo.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/govinfo.gov" }, { "rank": 1267, "domain": "messenger.com", "state": "found", "status": 200, "robotsTxt": "https://www.messenger.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/messenger.com" }, { "rank": 1268, "domain": "gwu.edu", "state": "found", "status": 200, "robotsTxt": "https://www.gwu.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gwu.edu" }, { "rank": 1269, "domain": "anu.edu.au", "state": "found", "status": 200, "robotsTxt": "https://www.anu.edu.au/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/anu.edu.au" }, { "rank": 1271, "domain": "ap.org", "state": "found", "status": 200, "robotsTxt": "https://ap.org/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "Amazonbot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "GoogleOther", "Meta-ExternalAgent", "Bytespider" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ap.org" }, { "rank": 1272, "domain": "torproject.org", "state": "found", "status": 200, "robotsTxt": "https://www.torproject.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/torproject.org" }, { "rank": 1273, "domain": "homedepot.com", "state": "found", "status": 200, "robotsTxt": "https://www.homedepot.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/homedepot.com" }, { "rank": 1274, "domain": "bayern.de", "state": "found", "status": 200, "robotsTxt": "https://www.bayern.de/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bayern.de" }, { "rank": 1276, "domain": "service-public.fr", "state": "found", "status": 200, "robotsTxt": "https://www.service-public.fr/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/service-public.fr" }, { "rank": 1278, "domain": "producthunt.com", "state": "found", "status": 200, "robotsTxt": "https://www.producthunt.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/producthunt.com" }, { "rank": 1279, "domain": "goethe.de", "state": "found", "status": 200, "robotsTxt": "https://www.goethe.de/robots.txt", "blocked": [ "PetalBot", "Bytespider", "SemrushBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/goethe.de" }, { "rank": 1280, "domain": "bleepingcomputer.com", "state": "found", "status": 200, "robotsTxt": "https://www.bleepingcomputer.com/robots.txt", "blocked": [ "ClaudeBot", "CCBot", "Applebot-Extended", "AhrefsBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/bleepingcomputer.com" }, { "rank": 1285, "domain": "craigslist.org", "state": "found", "status": 200, "robotsTxt": "https://craigslist.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/craigslist.org" }, { "rank": 1287, "domain": "nationalarchives.gov.uk", "state": "found", "status": 200, "robotsTxt": "https://www.nationalarchives.gov.uk/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/nationalarchives.gov.uk" }, { "rank": 1288, "domain": "foxbusiness.com", "state": "found", "status": 200, "robotsTxt": "https://www.foxbusiness.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/foxbusiness.com" }, { "rank": 1289, "domain": "cms.gov", "state": "found", "status": 200, "robotsTxt": "https://www.cms.gov/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/cms.gov" }, { "rank": 1291, "domain": "ijg.org", "state": "none", "status": 404, "robotsTxt": "https://ijg.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ijg.org" }, { "rank": 1295, "domain": "visa.com", "state": "found", "status": 200, "robotsTxt": "https://www.visa.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/visa.com" }, { "rank": 1296, "domain": "unsw.edu.au", "state": "found", "status": 200, "robotsTxt": "https://www.unsw.edu.au/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/unsw.edu.au" }, { "rank": 1298, "domain": "udel.edu", "state": "found", "status": 200, "robotsTxt": "https://www.udel.edu/content/dam/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/udel.edu" }, { "rank": 1299, "domain": "serverwatch.com", "state": "found", "status": 200, "robotsTxt": "https://www.serverwatch.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/serverwatch.com" }, { "rank": 1301, "domain": "html5up.net", "state": "found", "status": 200, "robotsTxt": "https://html5up.net/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/html5up.net" }, { "rank": 1302, "domain": "wikidot.com", "state": "none", "status": 404, "robotsTxt": "https://wikidot.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/wikidot.com" }, { "rank": 1303, "domain": "icloud.com", "state": "found", "status": 200, "robotsTxt": "https://www.icloud.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/icloud.com" }, { "rank": 1304, "domain": "uu.nl", "state": "found", "status": 200, "robotsTxt": "https://www.uu.nl/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/uu.nl" }, { "rank": 1305, "domain": "business-standard.com", "state": "refused", "status": 403, "robotsTxt": "https://www.business-standard.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/business-standard.com" }, { "rank": 1306, "domain": "wassenaar.org", "state": "found", "status": 200, "robotsTxt": "https://www.wassenaar.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/wassenaar.org" }, { "rank": 1307, "domain": "monday.com", "state": "found", "status": 200, "robotsTxt": "https://monday.com/robots.txt", "blocked": [], "partlyBlocked": [ "OAI-SearchBot", "PerplexityBot", "ClaudeBot", "Google-Extended", "GoogleOther", "ChatGPT-User", "Perplexity-User" ], "llmsTxt": true, "report": "https://canaicrawl.com/report/monday.com" }, { "rank": 1309, "domain": "verywellmind.com", "state": "found", "status": 200, "robotsTxt": "https://www.verywellmind.com/robots.txt", "blocked": [ "PetalBot", "Baiduspider", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "Amazonbot", "YouBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "Claude-User", "Perplexity-User", "MistralAI-User", "Meta-ExternalFetcher" ], "partlyBlocked": [ "OAI-SearchBot", "GPTBot", "ChatGPT-User" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/verywellmind.com" }, { "rank": 1312, "domain": "prweb.com", "state": "found", "status": 200, "robotsTxt": "https://www.prweb.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/prweb.com" }, { "rank": 1313, "domain": "gandi.net", "state": "found", "status": 200, "robotsTxt": "https://www.gandi.net/robots.txt", "blocked": [ "YandexBot", "AhrefsBot", "DotBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/gandi.net" }, { "rank": 1316, "domain": "scopus.com", "state": "refused", "status": 403, "robotsTxt": "https://scopus.com/robots.txt", "blocked": null, "partlyBlocked": null, "llmsTxt": null, "report": "https://canaicrawl.com/report/scopus.com" }, { "rank": 1317, "domain": "coinmarketcap.com", "state": "found", "status": 200, "robotsTxt": "https://coinmarketcap.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/coinmarketcap.com" }, { "rank": 1318, "domain": "github.blog", "state": "found", "status": 200, "robotsTxt": "https://github.blog/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/github.blog" }, { "rank": 1320, "domain": "ama-assn.org", "state": "found", "status": 200, "robotsTxt": "https://www.ama-assn.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/ama-assn.org" }, { "rank": 1321, "domain": "zdf.de", "state": "found", "status": 200, "robotsTxt": "https://www.zdf.de/robots.txt", "blocked": [], "partlyBlocked": [ "GPTBot", "Google-Extended", "CCBot", "ChatGPT-User", "SemrushBot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/zdf.de" }, { "rank": 1324, "domain": "adweek.com", "state": "found", "status": 200, "robotsTxt": "https://www.adweek.com/robots.txt", "blocked": [ "Baiduspider", "YouBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/adweek.com" }, { "rank": 1325, "domain": "colostate.edu", "state": "found", "status": 200, "robotsTxt": "http://www.colostate.edu/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/colostate.edu" }, { "rank": 1328, "domain": "freetds.org", "state": "none", "status": 404, "robotsTxt": "https://www.freetds.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/freetds.org" }, { "rank": 1329, "domain": "hwg.org", "state": "found", "status": 200, "robotsTxt": "https://hwg.org/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/hwg.org" }, { "rank": 1330, "domain": "pdflib.com", "state": "found", "status": 200, "robotsTxt": "https://www.pdflib.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/pdflib.com" }, { "rank": 1331, "domain": "straitstimes.com", "state": "found", "status": 200, "robotsTxt": "https://www.straitstimes.com/robots.txt", "blocked": [ "Google-Extended" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/straitstimes.com" }, { "rank": 1332, "domain": "rfi.fr", "state": "found", "status": 200, "robotsTxt": "https://www.rfi.fr/robots.txt", "blocked": [ "OAI-SearchBot", "PerplexityBot", "Claude-SearchBot", "DuckAssistBot", "YouBot", "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "ChatGPT-User", "Claude-User", "Perplexity-User", "MistralAI-User", "AhrefsBot", "SemrushBot", "DotBot" ], "partlyBlocked": [ "Googlebot", "Applebot" ], "llmsTxt": false, "report": "https://canaicrawl.com/report/rfi.fr" }, { "rank": 1333, "domain": "tagesschau.de", "state": "found", "status": 200, "robotsTxt": "https://www.tagesschau.de/robots.txt", "blocked": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "AhrefsBot" ], "partlyBlocked": [], "llmsTxt": false, "report": "https://canaicrawl.com/report/tagesschau.de" }, { "rank": 1334, "domain": "coinbase.com", "state": "found", "status": 200, "robotsTxt": "https://www.coinbase.com/robots.txt", "blocked": [], "partlyBlocked": [], "llmsTxt": true, "report": "https://canaicrawl.com/report/coinbase.com" } ] }