User-agent: * Disallow: /modules/captcha/security_codes/ Disallow: /breves/ Crawl-Delay: 1 # Custom disallows: User-agent: mail.ru_bot Disallow: / User-agent: Applebot Disallow: / User-agent: Amzn-SearchBot Disallow: / # Updated on 2026-05-05 by JC # AI Bots: https://crawlercheck.com/directory/ai-bots User-agent: ai2bot Allow: / User-agent: ai2bot-dolma Allow: / User-agent: big-sur-ai Allow: / User-agent: chatgpt-operator Allow: / User-agent: chatgpt-user Allow: / User-agent: chatgpt-user-2-0 Allow: / User-agent: claude-code Allow: / User-agent: claude-searchbot Allow: / User-agent: claude-user Allow: / User-agent: claude-web Allow: / User-agent: claudebot Allow: / User-agent: deepseekbot Allow: / User-agent: digitaloceangenai-crawler Allow: / User-agent: duckassistbot Allow: / User-agent: gptbot Allow: / User-agent: google-extended Allow: / User-agent: grok Allow: / User-agent: grokbot Allow: / User-agent: liner-bot Allow: / User-agent: mistralai-user Allow: / User-agent: mistralai-user-1-0 Allow: / User-agent: mycentralaiscraperbot Allow: / User-agent: oai-searchbot Allow: / User-agent: pangubot Allow: / User-agent: perplexity-user Allow: / User-agent: perplexity-user-1-0 Allow: / User-agent: perplexitybot Allow: / User-agent: sbintuitionsbot Allow: / User-agent: youbot Allow: / User-agent: anthropic-ai Allow: / User-agent: cohere-ai Allow: / User-agent: cohere-training-data-crawler Allow: / User-agent: img2dataset Allow: / User-agent: quillbot-com Allow: / # Cloud Services: https://crawlercheck.com/directory/cloud-services User-agent: aliyunsecbot Disallow: / User-agent: amazonbot Disallow: / User-agent: amzn-user Disallow: / User-agent: google-cloudvertexbot Disallow: / # Google Bots: https://crawlercheck.com/directory/google-bots User-agent: gemini-deep-research Allow: / User-agent: google-inspectiontool Allow: / User-agent: googleother Allow: / User-agent: googleother-image Allow: / User-agent: googleother-video Allow: / User-agent: googlebot-discovery Allow: / User-agent: googlebot-image Allow: / User-agent: googlebot-news Allow: / User-agent: googlebot-video Allow: / User-agent: storebot-google Allow: / # Other Agents: https://crawlercheck.com/directory/other-agents User-agent: turnitinbot Allow: / User-agent: viennatinybot Allow: / User-agent: archive-org-bot Allow: / User-agent: ia-archiver Allow: / User-agent: ia-archiver-web-archive-org Allow: / User-agent: peer39-crawler-1-0 Allow: / # SEO Tools: https://crawlercheck.com/directory/seo-tools User-agent: ahrefsbot Disallow: / User-agent: audigentadbot Disallow: / User-agent: awariorssbot Disallow: / User-agent: awariosmartbot Disallow: / User-agent: brandwatch Disallow: / User-agent: chrome-lighthouse Disallow: / User-agent: dataforseobot Disallow: / User-agent: dotbot Disallow: / User-agent: google-page-speed-insights Disallow: / User-agent: jetslide Disallow: / User-agent: mj12bot Disallow: / User-agent: meltwater Disallow: / User-agent: neticlebot Disallow: / User-agent: netvibes Disallow: / User-agent: screaming-frog-seo-spider Disallow: / User-agent: searchmetricsbot Disallow: / User-agent: semrushbot Disallow: / User-agent: semrushbot-ocob Disallow: / User-agent: semrushbotswa Disallow: / User-agent: sidetrade-indexer-bot Disallow: / User-agent: peer39-crawler Disallow: / # Scrappers: https://crawlercheck.com/directory/scrapers User-agent: amazon-kendra Disallow: / User-agent: arquivo-web-crawler Disallow: / User-agent: blexbot Disallow: / User-agent: barkrowler Disallow: / User-agent: bravest Disallow: / User-agent: bytespider Disallow: / User-agent: ccbot Disallow: / User-agent: cotoyogi Disallow: / User-agent: crawl4ai Disallow: / User-agent: crawlspace Disallow: / User-agent: diffbot Disallow: / User-agent: echobot-bot Disallow: / User-agent: echoboxbot Disallow: / User-agent: factset-spyderbot Disallow: / User-agent: firecrawl Disallow: / User-agent: friendlycrawler Disallow: / User-agent: icc-crawler Disallow: / User-agent: isscyberriskcrawler Disallow: / User-agent: imagesiftbot Disallow: / User-agent: jenkersbot Disallow: / User-agent: kangaroo-bot Disallow: / User-agent: livelapbot Disallow: / User-agent: mauibot Disallow: / User-agent: moodlebot Disallow: / User-agent: newsnow Disallow: / User-agent: novaact Disallow: / User-agent: poseidon-research-crawler Disallow: / User-agent: qualifiedbot Disallow: / User-agent: scrapy Disallow: / User-agent: seekportbot Disallow: / User-agent: seekr Disallow: / User-agent: seekrbot Disallow: / User-agent: taragroup-intelligent-bot Disallow: / User-agent: timpibot Disallow: / User-agent: turnitin Disallow: / User-agent: velenpublicwebcrawler Disallow: / User-agent: webzio-extended Disallow: / User-agent: coccocbot-web Disallow: / User-agent: crawler4j Disallow: / User-agent: hada-news Disallow: / User-agent: iaskspider Disallow: / User-agent: iaskspider-2-0 Disallow: / User-agent: imediaethics-org Disallow: / User-agent: imgproxy Disallow: / User-agent: magpie-crawler Disallow: / User-agent: netestate-imprint-crawler Disallow: / User-agent: news-please Disallow: / User-agent: omgili Disallow: / User-agent: omgilibot Disallow: / User-agent: yacy Disallow: / User-agent: yacybot Disallow: / # Social bots: https://crawlercheck.com/directory/social-bots User-agent: facebookbot Disallow: / User-agent: linkedinbot Disallow: / User-agent: meta-externalads Disallow: / User-agent: meta-externalagent Disallow: / User-agent: meta-externalfetcher Disallow: / User-agent: meta-webindexer Disallow: / User-agent: pinterestbot Disallow: / User-agent: quora-bot Disallow: / User-agent: slackbot Disallow: / User-agent: twitterbot Disallow: / User-agent: facebookexternalhit Disallow: / User-agent: meta-externalads-lowercase Disallow: / User-agent: meta-externalagent-lowercase Disallow: / User-agent: meta-externalfetcher-lowercase Disallow: / User-agent: meta-webindexer-lowercase Disallow: / # Limits: User-agent: DataForSeoBot Crawl-delay: 5 User-agent: dotbot Crawl-delay: 5 User-agent: barkrowler Crawl-Delay: 5 User-agent: tigerbot Crawl-Delay: 5 User-agent: linkbot Crawl-Delay: 5