<?php

/**
 * webtrees: online genealogy
 * Copyright (C) 2025 webtrees development team
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <https://www.gnu.org/licenses/>.
 */

declare(strict_types=1);

namespace Fisharebest\Webtrees\Http\Middleware;

use Fig\Http\Message\StatusCodeInterface;
use Fisharebest\Webtrees\Registry;
use Fisharebest\Webtrees\Services\NetworkService;
use Fisharebest\Webtrees\Validator;
use IPLib\Address\AddressInterface;
use IPLib\Factory;
use IPLib\Range\RangeInterface;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\ServerRequestInterface;
use Psr\Http\Server\MiddlewareInterface;
use Psr\Http\Server\RequestHandlerInterface;

use function array_filter;
use function array_map;
use function assert;
use function count;
use function gethostbyaddr;
use function gethostbyname;
use function preg_match_all;
use function random_int;
use function response;
use function str_contains;
use function str_ends_with;

/**
 * Middleware to block bad robots before they waste our valuable CPU cycles.
 */
class BadBotBlocker implements MiddlewareInterface
{
    public const ROBOT_ATTRIBUTE_NAME = 'is-a-robot';

    // Cache whois requests.  Try to avoid all caches expiring at the same time.
    private const WHOIS_TTL_MIN = 28 * 86400;
    private const WHOIS_TTL_MAX = 35 * 86400;

    // An opinionated list of "bad" robots. Typically, these are AI and SEO crawlers.
    // Taken from https://codeberg.org/fisharebest/robot-tools/src/branch/main/robots.php
    public const BAD_ROBOTS = [
        '007ac9 Crawler',
        '008',
        '2ip bot',
        '2ip.ru',
        '360monitoring',
        '360Spider',
        '5emeRue',
        '5erue',
        '7Siters',
        'A Patent Crawler',
        'A360-Search',
        'A6-Indexer',
        'aa',
        'AASA-Bot',
        'ABEvalBot',
        'Aboundex',
        'absearch-crawler',
        'AcademicBotRTU',
        'acapbot',
        'Accessible Web Bot',
        'AccessibleWebBot',
        'AccessStatus',
        'acoonbot',
        'Acquia optimize (Monsido)',
        'ActiveComply',
        'Acunetix Security Scanner',
        'Acunetix Web Vulnerability Scanner',
        'Adagio Bot',
        'Adagiobot',
        'adbeat-check-access-script',
        'adbeat_bot',
        'AdBot',
        'AddSearchBot',
        'AddThis',
        'adequat',
        'adidxbot',
        'AdkernelTopicCrawler',
        'admantx',
        'AdsBot-Google',
        'adscanner',
        'AdsTxtCrawler',
        'AdvBot',
        'Ahrefs Site Audit',
        'AhrefsBot',
        'AhrefsSiteAudit',
        'AI2Bot',
        'Ai2Bot-Dolma',
        'aiHitBot',
        'AISearchBot',
        'AlertSite',
        'Alertsite by Smartbear',
        'Alexa Archive',
        'alexa site audit',
        'Alexabot',
        'AlexandriaOrgBot',
        'Alexibot',
        'Algolia',
        'AliyunSecBot',
        'AllAfrica',
        'AlphaBot',
        'Amazon AdBot',
        'amazon-kendra',
        'AmazonAdBot',
        'Amazonbot',
        'AmazonBuyForMe',
        'AmazonProductDiscovery',
        'AmiSoftware',
        'AndersPinkBot',
        'Andibot',
        'Anomura',
        'anthropic-ai',
        'antibot',
        'AnyEvent',
        'Apercite',
        'AppInsights',
        'Applebot',
        'AppleNewsBot',
        'Aqua_Products',
        'arabot',
        'Archive-It',
        'archive.org_bot',
        'ArchiveBot',
        'Arquivo-web-crawler',
        'artemis web reader',
        'Ask n read',
        'asknread.com',
        'AspiegelBot',
        'asterias',
        'atlassian-bot',
        'Atom Feed Robot',
        'Attracta',
        'AudigentAdBot',
        'Audisto Crawler',
        'Augure',
        'auramundi',
        'Authory',
        'Automaton',
        'Automattic Analytics Crawler',
        'Automattic Feed Fetcher',
        'Awario',
        'awesomecrawler',
        'B2B Bot',
        'b2w',
        'BackDoorBot',
        'BacklinkCrawler',
        'BacklinksExtendedBot',
        'Baidu-ADS',
        'Baidu-YunGuanCe',
        'Baiduspider',
        'Barkrowler',
        'BazQux',
        'BDCbot',
        'bedrockbot',
        'BehloolBot',
        'BestChange',
        'betaBot',
        'Better Uptime',
        'Bibliotheque Nacional de France Crawler',
        'bidswitchbot',
        'BIGLOTRON',
        'bigsur.ai',
        'BigUpDataBot',
        'binarycanary',
        'Bing Ads',
        'Bing Preview',
        'BingPreview',
        'binlar',
        'Birdcrawlerbot',
        'BitBot',
        'bitlybot',
        'BitSightBot',
        'bl.uk_lddc_bot',
        'Black Duck Fast Dynamic',
        'Black Hole',
        'Blackboard',
        'Blekkobot',
        'BLEXBot',
        'Bling ERP',
        'BlingERP',
        'Blockaid',
        'blogmuraBot',
        'Blogtrottr',
        'BlogVault',
        'BlowFish',
        'BLP_bbot',
        'Bluesky',
        'bne.es_bot',
        'bnf.fr_bot',
        'BoardGamePrices Bot',
        'BomboraBot',
        'Bookmark search tool',
        'bot-pge.chlooe.com',
        'Bot.AraTurka.com',
        'BotALot',
        'botify',
        'BotRightHere',
        'BoxcarBot',
        'br-crawler',
        'brainobot',
        'BrandONbot',
        'BrandVerity',
        'Bravebot',
        'Brightbot',
        'BrightEdge Bot',
        'BrightEdge Crawler',
        'brokenlinkcheck.com',
        'BTWebClient',
        'BUbiNG',
        'BublupBot',
        'Buck',
        'Buddybot',
        'BuddyBot',
        'BufferLinkPreviewBot',
        'BugsNag',
        'Bugsnag script fetcher',
        'BuiltBotTough',
        'Bullseye',
        'BunnySlippers',
        'bushbaby',
        'Buttondown',
        'buzzbot',
        'Bytespider',
        'CaliberBot',
        'Caliperbot',
        'CandidateCrawler',
        'CapitalOneBot',
        'CapsuleChecker',
        'carbon-umbrella-bot',
        'careerbot',
        'Catchpoint',
        'caveman-hunter',
        'CC Metadata Scaper',
        'cc_bot',
        'CCBot',
        'CDSCbot',
        'Cegbfeieh',
        'CensysInspect',
        'CentComBot',
        'Centurybot',
        'CertChief',
        'cf-qualys-scanner',
        'changedetection',
        'channable',
        'Channel3Bot',
        'ChatGLM-Spider',
        'ChatGPT Agent',
        'ChatGPT-User',
        'Checkly',
        'CheckMarkNetwork',
        'CheeseBot',
        'CherryPicker',
        'Chirp',
        'Chrome-Lighthouse',
        'Cincraw',
        'cis5550-crawler',
        'Cision',
        'CISPA Webcrawler',
        'citeseerxbot',
        'Citoid',
        'Clarity-Bot',
        'ClarityBot',
        'Claude-SearchBot',
        'Claude-User',
        'Claude-Web',
        'ClaudeBot',
        'Clearscopebot',
        'Clickagy',
        'ClickUpLinkUnfurler',
        'Cliqzbot',
        'Cloudflare Custom Hostname Verification',
        'Cloudflare Radar URL Scanner',
        'Cloudflare SpeedTest',
        'Cloudflare Stream Webhook',
        'CloudFlare-AlwaysOnline',
        'Cloudflare-AutoRAG',
        'cloudflare-csup',
        'Cloudflare-Healthchecks',
        'CloudFlare-Prefetch',
        'Cloudflare-SSLDetector',
        'Cloudflare-Traffic-Manager',
        'Cloudflare-Validator',
        'CloudflareDiagnostics',
        'Cloudtrellis',
        'CloudVertexBot',
        'Cludo',
        'cludo.com',
        'CMS-Checker',
        'coccoc',
        'Cocolyzebot',
        'CodaBot',
        'coexel',
        'cognitiveSEO Crawler',
        'cohere-ai',
        'cohere-training-data-crawler',
        'Collapsify',
        'colly',
        'CommaFeed',
        'Companybook-Crawler',
        'content crawler spider',
        'contentking',
        'ContextAd Bot',
        'ContextualBot',
        'contxbot',
        'convera',
        'ConveraCrawler',
        'Convermax',
        'Cookie Hub',
        'cookie-maestro',
        'Cookiebot',
        'CookieHub',
        'CookieYesbot',
        'Copernic',
        'CopyRightCheck',
        'Corporama',
        'Cosmos',
        'Cotoyogi',
        'Coveo Bot',
        'Coveobot',
        'Cozi-iCalendar-FeedReader',
        'Crawl4AI',
        'crawler.freespoke.com',
        'crawler4j',
        'crawler_eb_germany',
        'Crawlson',
        'Crawlspace',
        'CrawlyProjectCrawler',
        'Crazy Egg',
        'Crescent',
        'CriteoBot',
        'Critical CSS Bot',
        'Criticalcss.com',
        'CrunchBot',
        'CrystalSemanticsBot',
        'Curebot',
        'Cutbot',
        'Cxense',
        'cXensebot',
        'CybaaBot',
        'CyberFindCrawler',
        'CyberPatrol',
        'CyotekWebCopy',
        'Cốc Cốc',
        'DareBoost',
        'Dark Visitor',
        'Datadog',
        'Datafeedwatch',
        'DataForSEO',
        'DataForSeoBot',
        'datagnionbot',
        'Datanyze',
        'Dataprovider',
        'Datenbank Crawler',
        'Daum',
        'Dazzle BlueSky Bot',
        'dbot',
        'dcrawl',
        'deadlinkchecker',
        'deepcrawl',
        'deepnoc',
        'DeepSeekBot',
        'Detectify',
        'Determ',
        'DeuSu',
        'Devin',
        'Diffbot',
        'Digg Deeper',
        'Digicert DCV',
        'DigiCert DCV Bot',
        'Digimind',
        'Digincore bot',
        'discobot',
        'Discordbot',
        'Disqus',
        'DittoSpyder',
        'DLC',
        'dlvr.it',
        'DMBrowser',
        'DnBCrawler-Analytics',
        'DNSScanner',
        'DnyzBot',
        'Do Not Track Verifier',
        'Doctom Monitor',
        'Domain Re-Animator Bot',
        'DomainCrawler',
        'Domains Project',
        'DomainStatsBot',
        'DotBot',
        'Dow Jones Searchbot',
        'Download Ninja',
        'Dragonbot',
        'Drata Autopilot',
        'Dratabot',
        'DreamHost Data Team',
        'drupact',
        'Drupalbot',
        'ds9',
        'Dubbotbot',
        'DuckAssistBot',
        'DuckDuckBot',
        'DuckDuckGo-Favicons-Bot',
        'DVbot',
        'e.ventures Investment Crawler',
        'EasyBib AutoCite',
        'easybill-ImportManager',
        'easyDNS Monitoring',
        'ec2linkfinder',
        'Echobot Bot',
        'EchoboxBot',
        'EcoVadisSustainabilityBot',
        'edisterbot',
        'electricmonk',
        'Elisabot',
        'ellisphere',
        'elmah.io Uptime Monitoring',
        'elmahio-uptimebot',
        'EmailCollector',
        'EmailSiphon',
        'EmailWolf',
        'Embedly',
        'emetriqContextualBot',
        'eMoney Advisor',
        'eMoneyBot',
        'epicbot',
        'EpivozCrawler',
        'eright',
        'ErisBot',
        'EroCrawler',
        'EtaoSpider',
        'europarchive.org',
        'ev-crawler',
        'evc-batch',
        'EveryoneSocialBot',
        'EvoUptimeBot',
        'Exabot',
        'Expanse',
        'Experibot',
        'ExteContextCrawl',
        'ExtLinksBot',
        'ExtractorPro',
        'Eyeotabot',
        'EZID',
        'EzLynx',
        'EzoicBot',
        'Ezooms',
        'facebook',
        'FacebookBot',
        'Facebot',
        'factset_spyderbot',
        'Factset_spyderbot',
        'FairAd Client',
        'FAST Enterprise Crawler',
        'FAST-WebCrawler',
        'FastDAST',
        'Fastmail Bot',
        'FastmailUA',
        'FDL Stats Bot',
        'Fedicabot',
        'FediDB',
        'FediIndex',
        'FediList Agent',
        'Fedineko',
        'fedoraplanet',
        'FedReporter Bot for FFIEC',
        'FedReporterDataBot',
        'Feedbin',
        'feedbot',
        'FeedBurner',
        'FeedDiscovery',
        'Feeder',
        'feeder.co',
        'Feedfetcher-Google',
        'FeedFlow',
        'Feedly',
        'Feedsearch Bot',
        'Feedsearch-Crawler',
        'Feedspot',
        'Feedwind',
        'FeedWind Crawler',
        'FemtosearchBot',
        'Fever',
        'FindITAnswersbot',
        'findlink',
        'findthatfile',
        'findxbot',
        'fiperbot',
        'FirecrawlAgent',
        'Flaming AttackBot',
        'Flamingo_SearchEngine',
        'FlipboardProxy',
        'FlipboardRSS',
        'fluffy',
        'fluid',
        'Flyriverbot',
        'Foobot',
        'Foregenix',
        'fr-crawler',
        'Freespoke',
        'FreeWebMonitoring SiteChecker',
        'Freshbot',
        'Freshping',
        'FreshRSS',
        'Friendica',
        'FriendlyCrawler',
        'fuelbot',
        'FullStory',
        'Funnelback',
        'FyndSearchEngine-Crawler',
        'FyndSearchEngine-ReCrawler',
        'Fyrebot',
        'g00g1e.net',
        'G2 Web Services',
        'g2reader-bot',
        'Gaisbot',
        'GarlikCrawler',
        'GeedoProductSearch',
        'Gemini-Deep-Research',
        'Genieo',
        'GenomeCrawlerd',
        'GetRight',
        'Ghost Inspector',
        'gigablast',
        'Gigabot',
        'GingerCrawler',
        'Gluten Free Crawler',
        'gnam gnam spider',
        'GnowitNewsbot',
        'Goodreads',
        'Google AdsBot',
        'Google AdSense',
        'Google Digital Asset Links',
        'Google Favicon',
        'Google Feed Fetcher',
        'Google Image Proxy',
        'Google Images',
        'Google Inspection Tool',
        'Google Publisher Center',
        'Google Schema Markup Testing Tool',
        'Google Scholar',
        'Google StoreBot',
        'Google Trust Services (DCV Check)',
        'Google Videos',
        'Google Web Preview',
        'Google-Ads-Creatives-Assistant',
        'Google-AdWords-Express',
        'Google-Adwords-Instant',
        'Google-Calendar-Importer',
        'Google-Certificates-Bridge',
        'Google-CloudVertexBot',
        'Google-Display-Ads-Bot',
        'Google-Extended',
        'Google-Firebase',
        'Google-InspectionTool',
        'Google-NotebookLM',
        'Google-PageRenderer',
        'Google-PhysicalWeb',
        'Google-Read-Aloud',
        'Google-Safety',
        'Google-Site-Verification',
        'Google-Structured-Data-Testing',
        'Google-Trust-Services',
        'google-xrawler',
        'GoogleAgent-Mariner',
        'GoogleAssociationService',
        'Googlebot-IA',
        'Googlebot-Image',
        'Googlebot-Mobile',
        'Googlebot-News',
        'Googlebot-Video',
        'GoogleDocs',
        'GoogleImageProxy',
        'GoogleOther',
        'GoogleProducer',
        'GoParserBot',
        'Gowikibot',
        'GPTBot',
        'grafana',
        'Grafana\'s Synthetic Monitoring',
        'Grammarly',
        'grapeshot',
        'GrapeshotCrawler',
        'Greppr Web Crawler',
        'Grobbot',
        'GroovinaAdsbot',
        'GroupHigh',
        'grub-client',
        'grub.org',
        'gsa-crawler',
        'gslfbot',
        'GTmetrix',
        'GuestpostsBot',
        'Gwene',
        'HaloBot',
        'HanaleiBot',
        'HaosouSpider',
        'Hardenize',
        'Harvest',
        'Hatena',
        'HawaiiBot',
        'HelloWork',
        'HelloworkJobPostingBot',
        'heritrix',
        'HetrixTools',
        'hgfAlphaXCrawl',
        'HIFIBot',
        'Honeybadger Uptime Check',
        'HoneybadgerBot',
        'HostTracker',
        'Hotjar',
        'hstspreload-bot',
        'HTTP Banner Detection',
        'HTTrack',
        'HubSpot',
        'Huckabot',
        'Huckabuy Bot',
        'humanlinks',
        'Hype Machine',
        'hypestat',
        'HyScore',
        'hyscore.io',
        'ia_archiver',
        'IABot',
        'IAS crawler',
        'ias-ie',
        'ias-va',
        'iAskBot',
        'iaskspider',
        'IbouBot',
        'ICBot',
        'ICC-Crawler',
        'ichiro',
        'Iframely',
        'IFTTT',
        'ImagesiftBot',
        'imageSpider',
        'img2dataset',
        'Impact.com Agent',
        'imrbot',
        'IndeedBot',
        'IndeedJobBot',
        'INETDEX-Bot',
        'InfoNaviRobot',
        'infoobot',
        'infoseek',
        'Innguma',
        'Innologica',
        'Inoreader',
        'inoreader.com',
        'Instapaper',
        'Instaparser',
        'integromedb',
        'intelium_bot',
        'intelx.io_bot',
        'InteractiveAdvertisingBureau',
        'InterfaxScanBot',
        'Internet Archive',
        'InternetArchiveBot',
        'InternetMeasurement',
        'IonCrawl',
        'ip-web-crawler.com',
        'ips-agent',
        'IRLbot',
        'Iron33',
        'IsDownBot',
        'iskanie',
        'IsraBot',
        'ISSCyberRiskCrawler',
        'istellabot',
        'it2media-domain-crawler',
        'Jagged Pixel UptimeBot',
        'James BOT',
        'JamesBOT',
        'Jamie\'s Spider',
        'JenkersBot',
        'JennyBot',
        'Jetbot',
        'jetmon',
        'Jetpack',
        'Jetty',
        'JikeSpider',
        'JobboerseBot',
        'jobswithgptcom-bot',
        'Jooblebot',
        'jpg-newsbot',
        'Jugendschutzprogramm-Crawler',
        'Jumio',
        'jyxobot',
        'k2spider',
        'K7MLWCBot',
        'Kagibot',
        'kaikki.org-digital-archive',
        'Kangaroo Bot',
        'KargoBot-Artemis',
        'kb.dk_bot',
        'kbcrawl',
        'Kemvibot',
        'Kenjin Spider',
        'Keydrop.io',
        'keys-so-bot',
        'Keyword Density',
        'kinsta',
        'Klaviyo',
        'Knowings',
        'Known Agent',
        'KomodiaBot',
        'Konqueror',
        'KosmioBot',
        'KrawlerBot',
        'KStandBot',
        'KunatoCrawler',
        'l9scan',
        'laion-huggingface-processor',
        'LAIONDownloader',
        'LamarkBot',
        'Landau-Media-Spider',
        'larbin',
        'Laserlikebot',
        'lb-spider',
        'LCC',
        'leadbox',
        'LegalMonster',
        'Leikibot',
        'Let\'s Encrypt',
        'Level9SearchBot',
        'LexiBot',
        'Library Of Congress Web Archiving',
        'libWeb',
        'LightspeedSystemsCrawler',
        'LinerBot',
        'Linespider',
        'Linguee Bot',
        'linkapediabot',
        'LinkArchiver',
        'LinkCheck',
        'linkchecker.pro',
        'linkdex',
        'LinkedInBot',
        'LinkextractorPro',
        'linkfluence',
        'LinkisBot',
        'linko',
        'LinkpadBot',
        'LinkScan',
        'LinksIndexerBot',
        'LinkTiger',
        'LinkupBot',
        'LinkWalker',
        'lipperhey',
        'LivelapBot',
        'lkxscan',
        'LMArenaUnfurlBot',
        'LNSpiderguy',
        'logicmonitor',
        'LogicMonitor SiteMonitor',
        'LoomlyBot',
        'lssbot',
        'lssrocketcrawler',
        'ltx71',
        'Luminator-robots',
        'lwp-trivial',
        'MaCoCu',
        'Macrobondbot',
        'MagiBot',
        'Magnet.me-web',
        'MagnetmeBot',
        'magpie-crawler',
        'MagpieRSS',
        'Magus Bot',
        'Mail.RU_Bot',
        'MailChimp',
        'MailRUBot',
        'MainWP',
        'MakeMerryBot',
        'Manus-User',
        'mappydata',
        'Marginalia Search',
        'MarketGoo',
        'Mars Finder',
        'Mastodon',
        'Mata Hari',
        'MatchorySearch',
        'MauiBot',
        'Mavifinds',
        'MBCrawler',
        'Medialogia Bot',
        'MedialogiaBot',
        'MediaMonitoringBot',
        'Mediapartners-Google',
        'Mediatoolkitbot',
        'Mediavine Medatada Parser',
        'MediavineMetadataParser',
        'MegaIndex',
        'Meltawer',
        'Meltwater',
        'memorybot',
        'mention',
        'meta-externalads',
        'meta-externalagent',
        'Meta-ExternalAgent',
        'meta-externalfetcher',
        'Meta-ExternalFetcher',
        'meta-webindexer',
        'MetadataScraper',
        'MetaInspector',
        'MetaJobBot',
        'MetaURI',
        'MgidBot',
        'Microsoft Preview',
        'MicrosoftPreview',
        'MIIxpc',
        'mindUpBot',
        'minicrawler',
        'Miniflux',
        'Minoru\'s Fediverse Crawler',
        'mirrorweb',
        'MirrorWebCrawler',
        'Missinglettr Bot',
        'MissinglettrBot',
        'Mister PiX',
        'MistralAI-User',
        'mithril-crawler',
        'MixnodeCache',
        'MixrankBot',
        'MJ12bot',
        'mlbot',
        'moatbot',
        'ModatScanner',
        'moget',
        'mojeek',
        'MojeekBot',
        'MonitageUptime',
        'monitis',
        'monitoring360',
        'MonitoRSS',
        'Monsidobot',
        'MonSpark',
        'montastic-monitor',
        'MontasticMonitor',
        'MoodleBot',
        'Moreover',
        'MotoMinerBot',
        'Moz dotbot',
        'Moz rogerbot',
        'MRGbot',
        'MS Search 4.0 Robot',
        'MS Search 6.0 Robot',
        'MSIECrawler',
        'MSN',
        'msnbot',
        'msrbot',
        'MTRobot',
        'MuckRack',
        'Multiviewbot',
        'MxToolbox',
        'MyCentralAIScraperBot',
        'MyEducationalCrawler',
        'mytwip',
        'NanoInteractive',
        'NAVER Blog Rssbot',
        'NaverBot',
        'Neevabot',
        'NerdByNature.Bot',
        'NerdyBot',
        'NetAnts',
        'netarkivindsamling',
        'Netcraft',
        'netEstate Imprint Crawler',
        'netEstate NE Crawler',
        'Neticle Crawler',
        'NetMechanic',
        'NetResearchServer',
        'NetSeer crawler',
        'NetSystemsResearch',
        'netumo',
        'Netvibes',
        'NETVIGIE',
        'New Relic',
        'New York Times Newsgathering',
        'NewRelicbot',
        'newsai',
        'NewsBlur',
        'newsharecounts',
        'NewsNow',
        'newspaper',
        'Newzbin',
        'NextCloud',
        'NextGenSearchBot',
        'Nicecrawler',
        'NICErsPRO',
        'niki-bot',
        'NimbleCrawler',
        'Nimbostratus-Bot',
        'NINJA bot',
        'Nitro-',
        'NitroBot',
        'NIXStatsbot',
        'NixStatsMonitoringBot',
        'NLUX_IAHarvester',
        'Nmap Scripting Engine',
        'NoahBot',
        'NodePing',
        'Noibu',
        'noorobot',
        'Nooshub',
        'NostoCrawlerBot',
        'Notabot',
        'NotebookLM',
        'NovaAct',
        'NPBot',
        'NTENTbot',
        'Nutch',
        'Nuzzel',
        'nyt_scraping',
        'OAI-SearchBot',
        'Observer',
        'Odin',
        'OdklBot',
        'officestorebot',
        'Offline Explorer',
        'Oh Dear',
        'OhDear',
        'omgili',
        'Omgilibot',
        'OnCrawl',
        'OneTrust',
        'Onetrust CMP Scanner',
        'online-webceo-bot',
        'OnlineOrNot',
        'OpenAI',
        'Openbot',
        'openfind',
        'Openfind data gatherer',
        'OpenGraph.io',
        'OpenGraphCheck',
        'OpenHoseBot',
        'OpenindexSpider',
        'OpenRSS',
        'OpenTheBoxBot',
        'Operator',
        'opinion-tracker',
        'Oracle Ultra Search',
        'OrangeBot',
        'Orbbot',
        'Orlo-LinkPreview',
        'Orthogaffe',
        'outbrain',
        'OutclicksBot',
        'OutsellURLValidator',
        'Overcast',
        'Owler',
        'OWLer-W',
        'Ozon Web Grabber',
        'page-preview-tool',
        'Page2RSS',
        'PagePeeker',
        'PageThing',
        'Pandalytics',
        'PanguBot',
        'Panopta',
        'Panscient',
        'panscient.com',
        'Paqlebot',
        'parisbot',
        'parse.ly',
        'parsely.com',
        'PDFBot',
        'peer39_crawler',
        'PerMan',
        'Perplexity-User',
        'PerplexityBot',
        'PetalBot',
        'PhindBot',
        'PiBot',
        'Pingdom',
        'PingPing Bot',
        'pingping.io',
        'Pinterest',
        'PiplBot',
        'Pixalate.com',
        'PlagAwareBot',
        'Plesk screenshot bot',
        'Pocket Casts Feed Parser',
        'PocketParser',
        'PodchaserParser',
        'Poduptime',
        'Poggio-Citations',
        'Poseidon Research Crawler',
        'postrank',
        'Potions',
        'PR-CY.RU',
        'Prerender',
        'PressEngine Bot',
        'PressEngineBot',
        'PriEcoBot',
        'Primalbot',
        'PrivacyAwareBot',
        'Pro Sitemaps',
        'productsup.io/crawler',
        'ProjectShield Url Check',
        'ProjectShield-UrlCheck',
        'ProPowerBot',
        'Protopage',
        'ProWebWalker',
        'proxem',
        'proximic',
        'PRTGCloudBot',
        'psbot',
        'PubMatic Crawler Bot',
        'PulsePoint',
        'pulsetic',
        'purebot',
        'PWABuilderHttpAgent',
        'qcbot',
        'QualifiedBot',
        'Qualys',
        'Quantcastbot',
        'QueryN Metasearch',
        'QuillBot',
        'quillbot.com',
        'Quora-Bot',
        'Qwam content intelligence',
        'Qwantbot-official',
        'Qwantify',
        'Qwarrybot',
        'Rackspace',
        'Radiation Retriever 1.1',
        'Rakuten Image extraction bot',
        'rakutenusabot-image',
        'RankActiveLinkBot',
        'RankFlex',
        'RasaBot',
        'rawweb-bot',
        'Readable',
        'redditbot',
        'Reelevant',
        'Refindbot',
        'RegionStuttgartBot',
        'RepoMonkey',
        'research/recap',
        'Retool',
        'RetrevoPageAnalyzer',
        'RetroListeCOM',
        'ReverseEngineeringBot',
        'RevvimGort',
        'reward-gateway',
        'RidderBot',
        'Riddler',
        'Rigor',
        'Rivva',
        'robots-ai-permissions',
        'Robozilla',
        'rogerbot',
        'RSiteAuditor',
        'RSS API',
        'rss-is-dead.lol web bot',
        'rss2tg bot',
        'RSSAPI',
        'RSSBot',
        'RSSingBot',
        'RukiCrawler',
        'RuxitSynthetic',
        'RyteBot',
        's4a',
        'SafeDNSBot',
        'SafeSearch microdata crawler',
        'SalesViewerBot',
        'Sansec Security Monitor',
        'SBIntuitionsBot',
        'SBL-BOT',
        'Schema-Markup-Validator',
        'scoop.it',
        'scoopit-crawler',
        'Scope3',
        'score3',
        'scour.ing',
        'ScourRSSBot',
        'ScoutJet',
        'scraping@nytimes.com',
        'Scrapy',
        'Screaming Frog SEO Spider',
        'scribdbot',
        'Scrubby',
        'Scrunchbot',
        'SE Ranking Bot',
        'SearchAtlas',
        'SearchmetricsBot',
        'searchpreview',
        'SEBot-WA',
        'SecurityHeaders',
        'Seekbot',
        'Seekport',
        'Seekr',
        'seewithkids',
        'semantic-visions.com crawler',
        'Semanticbot',
        'SemanticScholarBot',
        'sempi.tech',
        'SemrushBot',
        'SentiBot',
        'Sentry',
        'Senutobot',
        'seo-audit-check-bot',
        'seo4ajax',
        'SeobilityBot',
        'SEOkicks',
        'SEOlizer',
        'SeoLyt',
        'seoscanners',
        'seostar.co',
        'SEOstats',
        'SequelWP',
        'SERankingBacklinksBot',
        'serendeputybot',
        'serpstatbot',
        'Server Density',
        'ServerHunterSpider',
        'Seznam',
        'SFDC-Callout',
        'ShapBot',
        'ShellBot',
        'ShortPixel',
        'Shortwave Image Fetcher',
        'Sidetrade indexer bot',
        'Silktide',
        'simplecrawler',
        'SimplePie',
        'SimpleScraper',
        'Sindup',
        'SirdataBot',
        'sistrix crawler',
        'site24x7',
        'SiteAuditBot',
        'SiteBot',
        'Sitebulb',
        'SiteCheck-sitecrawl',
        'sitecheck.internetseer.com',
        'SiteCheckerBotCrawler',
        'siteexplorer.info',
        'siteimprove',
        'Siteimprove Crawl',
        'Siteimprove.com',
        'SiteLock',
        'SiteSearch360',
        'SiteSnagger',
        'SitesOverPagesBot',
        'SiteSucker',
        'SiteUptime',
        'SiteUpTimeBot',
        'Skroutz ImageBot',
        'SkroutzBot',
        'Skype',
        'Slack Image Proxy',
        'Slack-ImgProxy',
        'Slackbot',
        'Sleuth',
        'SlickBot',
        'slickstream',
        'slurp',
        'SmarshBot',
        'SmartologyBot',
        'SMTBot',
        'SMTnetPMBot',
        'Snap URL Preview Service',
        'Snapchat',
        'SocialRankIOBot',
        'Sogou',
        'SolarWinds Observability',
        'Sonic',
        'Sosospider',
        'SottopopNone',
        'SpankBot',
        'spanner',
        'spbot',
        'special_archiver',
        'Spectate',
        'SpeedyIndexBot',
        'Spider',
        'Spinn3r',
        'Splunk',
        'spotter',
        'SputnikBot',
        'SQWatcher',
        'SSL Labs',
        'Stape',
        'star-finder.de Bot',
        'StartmeBot',
        'Statabot',
        'StatistikAustria',
        'StatsDroneBot',
        'statuscake',
        'StatusNestBacklinkSpider',
        'SteamChat',
        'stepstoneCrawlBot',
        'Storebot-Google',
        'StorygizeBot',
        'StractBot',
        'Streamline3Bot',
        'Sucuri',
        'SummalyBot',
        'Summify',
        'SuperBot',
        'Superfeedr',
        'superpagesbot2.0',
        'SurdotlyBot',
        'SurveyBot',
        'suzuran',
        'sv-watchagent',
        'Swiftbot',
        'Swifteq Link Checker',
        'SwifteqLinkChecker',
        'SWIMGBot',
        'Synapse',
        'SynthesiBot',
        'Synthesio',
        'Synthetic Bot',
        'synthetic-monitoring-agent',
        'Sysomos',
        'Szukacz',
        't3versionsBot',
        'Taboola',
        'tagoobot',
        'Talkwater',
        'TangibleeBot',
        'TaraGroup Intelligent Bot',
        'TavilyBot',
        'Telegram Bot',
        'TelegramBot',
        'Teleport',
        'Telesoft',
        'Teoma',
        'Termly',
        'TerraCotta',
        'Testomatobot',
        'The Intraformant',
        'TheNomad',
        'theoldreader.com',
        'Thinkbot',
        'Thinklab',
        'Thousand Eyes Cloud Agent',
        'TigerBot',
        'TikTokSpider',
        'Timpibot',
        'TinEye',
        'Tiny Tiny RSS',
        'Titan',
        'TLS tester',
        'toCrawl',
        'TombaPublicWebCrawler',
        'Toplistbot',
        'ToutiaoSpider',
        'Traackr.com',
        'tracemyfile',
        'trafilatura',
        'Trellis-Services',
        'trendeo',
        'Trendiction Bot',
        'trendictionbot',
        'trendkite-akashic-crawler',
        'TrendsmapResolver',
        'trendybuzz',
        'trovitBot',
        'True_Robot',
        'TruliaBot',
        'TSMbot',
        'TTD-Content',
        'Tumblr',
        'turingos',
        'Turnitin',
        'TweetedTimes',
        'TweetmemeBot',
        'twengabot',
        'TwinAgent',
        'Twingly',
        'TwinWaveScanner',
        'Twitterbot',
        'Twurly',
        'UbiCrawler',
        'UGAResearchAgent',
        'uipbot',
        'um-FC',
        'um-IC',
        'um-LN',
        'upday',
        'updown.io',
        'UpDownBot',
        'Updownerbot',
        'Upflow',
        'Uptime',
        'uptimedoctor',
        'Uptimia',
        'uptrends',
        'UptrendsBot',
        'URL Control',
        'URL_Spider_Pro',
        'urlappendbot',
        'urlsuma.de crawler',
        'URLy Warning',
        'usasearch',
        'UsineNouvelleCrawler',
        'UT-Dorkbot',
        'VCI',
        'vebidoobot',
        'vecteurplus',
        'VelenPublicWebCrawler',
        'Veoozbot',
        'Verispider',
        'Verity',
        'verticalsearch',
        'Viber',
        'videootv Bot',
        'videootvBot',
        'Vigil',
        'virustotal',
        'VKRobot',
        'vkShare',
        'VoilaBot',
        'voltron',
        'VoluumDSP-content-bot',
        'vsw',
        'vuhuvBot',
        'W3 Validator Services',
        'W3C-mobileOK',
        'W3C_CSS_Validator',
        'W3C_Unicorn',
        'WadooBot',
        'WanScannerBot',
        'WARDBot',
        'WASALive-Bot',
        'Watchbot',
        'Watchful',
        'wbsearchbot',
        'Web Image Collector',
        'web-archive-net.com.bot',
        'WebAuto',
        'WebBandit',
        'WebCapture 2.0',
        'webcompanycrawler',
        'WebCopier',
        'WebCrawler',
        'WebDataStats',
        'WebEnhancer',
        'WebmasterWorldForumBot',
        'webmon ',
        'weborama-fetcher',
        'webpagetest',
        'WebReaper',
        'WebSauger',
        'Website Quester',
        'Website-info.net-Robot',
        'WebsiteOps',
        'websitepulse checker',
        'WebsitePulseBot',
        'webspidermount',
        'WebStripper',
        'webzio',
        'Webzio-Extended',
        'WebZIP',
        'WellKnownBot',
        'WepchSearchEngine',
        'WhatsApp',
        'WikiDo',
        'WincherBot',
        'winello',
        'WinHTTrack',
        'WireReaderBot',
        'WiseGuys Robot',
        'wknd-bot',
        'WMF Zotero Translation Server',
        'wocbot',
        'woobot',
        'woorankreview',
        'WordCountBot',
        'WordupInfoSearch',
        'woriobot',
        'WormlyBot',
        'wotbox',
        'WOVN Crawler',
        'WovnCrawler',
        'wpbot',
        'WPMU DEV Broken Link Checker',
        'WPMU DEV Hub',
        'wpmudev',
        'WPMUDEV Uptime Monitor',
        'WPUmbrella',
        'WRTNBot',
        'ws-bot-v1',
        'WSM',
        'WTotem',
        'WWW-Collector-E',
        'WWW-Mechanize',
        'www.deadlinkchecker.com',
        'www.uptime.com',
        'xenU',
        'Xenu Link Sleuth',
        'Xenu\'s',
        'Xing Bot',
        'XML Sitemaps Generator',
        'xovibot',
        'XoviOnpageCrawler',
        'Xpanse-bot',
        'XY-Archive-Compliance',
        'Y!J-ASR',
        'Y!J-BRW',
        'Y!J-BRZ/YATSHA crawler',
        'Y!J-DLC',
        'yacybot',
        'YaDirectFetcher',
        'Yahoo Ad monitoring',
        'Yahoo Japan',
        'Yahoo Link Preview',
        'Yahoo Pipes 1.0',
        'Yahoo! Slurp',
        'YahooCacheSystem',
        'YahooMailProxy',
        'YaK',
        'yandex',
        'YandexAdditional',
        'YandexBot',
        'YandexImages',
        'YandexMobileBot',
        'YandexRenderResourcesBot',
        'YandexVertis',
        'YandexVideo',
        'YandoriRSSBot',
        'yanga',
        'Yellowbrandprotectionbot',
        'YelpBot',
        'Yeti',
        'Yext Inc',
        'YextBot',
        'YisouSpider',
        'yoozBot',
        'YouBot',
        'YoudaoBot',
        'Youmag',
        'Zabbix',
        'ZanistaBot',
        'Zao',
        'Zealbot',
        'zenback bot',
        'zeus',
        'Zeus Link Scout',
        'zgrab',
        'Zite',
        'Zoombot',
        'ZoomInfo',
        'ZoominfoBot',
        'ZoteroTranslationServer',
        'ZumBot',
        'ZuperlistBot',
        'ZyBorg',
    ];

    /**
     * Some search engines use reverse/forward DNS to verify the IP address.
     *
     * @see https://support.google.com/webmasters/answer/80553?hl=en
     * @see https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0
     * @see https://www.bing.com/webmaster/help/how-to-verify-bingbot-3905dc26
     * @see https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html
     * @see https://www.mojeek.com/bot.html
     */
    private const ROBOT_REV_FWD_DNS = [
        'BingPreview'      => ['.search.msn.com'],
        'Google'           => ['.google.com', '.googlebot.com'],
        'Mail.RU_Bot'      => ['.mail.ru'],
        'MicrosoftPreview' => ['.search.msn.com'],
        'MojeekBot'        => ['.mojeek.com'],
        'Qwantify'         => ['.qwant.com'],
        'Sogou'            => ['.crawl.sogou.com'],
        'Yahoo'            => ['.crawl.yahoo.net'],
        'Yandex'           => ['.yandex.ru', '.yandex.net', '.yandex.com'],
        'bingbot'          => ['.search.msn.com'],
        'msnbot'           => ['.search.msn.com'],
    ];

    /**
     * Some search engines only use reverse DNS to verify the IP address.
     *
     * @see https://help.baidu.com/question?prod_id=99&class=0&id=3001
     * @see https://napoveda.seznam.cz/en/full-text-search/seznambot-crawler
     * @see https://www.ionos.de/terms-gtc/faq-crawler
     */
    private const ROBOT_REV_ONLY_DNS = [
        'Baiduspider' => ['.baidu.com', '.baidu.jp'],
        'FreshBot'    => ['.seznam.cz'],
        'Neevabot'    => ['.neeva.com'],
        'SeznamBot'   => ['.seznam.cz'],
    ];

    /**
     * Some search engines operate from designated IP addresses.
     * TODO: fetch current lists of IPs, rather than use hard-coded values.
     * See https://merj.com/blog/dont-block-what-you-want-duckduckgo-and-common-crawl-to-provide-ip-address-api-endpoints
     */

    /**
     * Some search engines operate from within a designated autonomous system.
     *
     * @see https://developers.facebook.com/docs/sharing/webmasters/crawler
     * @see https://www.facebook.com/peering/
     */
    private const ROBOT_ASNS = [
        'facebook' => ['AS32934'],
        'twitter'  => ['AS13414'],
    ];

    private NetworkService $network_service;

    public function __construct(NetworkService $network_service)
    {
        $this->network_service = $network_service;
    }

    public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
    {
        $ua      = Validator::serverParams($request)->string('HTTP_USER_AGENT', '');
        $ip      = Validator::attributes($request)->string('client-ip');
        $address = Factory::parseAddressString($ip);
        assert($address instanceof AddressInterface);

        if ($ua === '') {
            return $this->response('Not acceptable: no-ua');
        }

        foreach (self::BAD_ROBOTS as $robot) {
            if (str_contains($ua, $robot)) {
                return $this->response('Not acceptable: bad-ua');
            }
        }

        $validated_bot =  false;

        foreach (self::ROBOT_REV_FWD_DNS as $robot => $valid_domains) {
            if (str_contains($ua, $robot)) {
                if ($this->checkRobotDNS($ip, $valid_domains, false)) {
                    $validated_bot = true;
                } else {
                    return $this->response('Not acceptable: bad-dns');
                }
            }
        }

        foreach (self::ROBOT_REV_ONLY_DNS as $robot => $valid_domains) {
            if (str_contains($ua, $robot)) {
                if ($this->checkRobotDNS($ip, $valid_domains, true)) {
                    $validated_bot = true;
                } else {
                    return $this->response('Not acceptable: bad-dns');
                }
            }
        }

        // TODO: fetch current lists of IPs, rather than use hard-coded values.

        foreach (self::ROBOT_ASNS as $robot => $asns) {
            foreach ($asns as $asn) {
                if (str_contains($ua, $robot)) {
                    foreach ($this->fetchIpRangesForAsn($asn) as $range) {
                        if ($range->contains($address)) {
                            $validated_bot = true;
                            continue 2;
                        }
                    }

                    return $this->response('Not acceptable: bad-dns');
                }
            }
        }

        // Allow sites to block access from entire networks.
        $block_asn = Validator::attributes($request)->string('block_asn', '');
        preg_match_all('/(AS\d+)/', $block_asn, $matches);

        foreach ($matches[1] as $asn) {
            foreach ($this->fetchIpRangesForAsn($asn) as $range) {
                if ($range->contains($address)) {
                    return $this->response('Not acceptable: bad-asn');
                }
            }
        }

        // No Cookies?  Few headers?  Probably a robot.
        $has_cookies     = $request->getCookieParams() !== [];
        $has_few_headers = count($request->getHeaders()) <= 11;
        $suspected_bot   = !$has_cookies && $has_few_headers;

        // Robots often claim to be a browser.
        $claims_to_be_human =
            str_contains($ua, 'Chrome/') ||
            str_contains($ua, 'Firefox/') ||
            str_contains($ua, 'Opera/') ||
            str_contains($ua, 'Safari/')
        ;

        // Validated bots (such as google and bing) use headless browsers.  This is OK.
        // Anyone else claiming to be a browser needs to prove it by setting a cookie.
        if (!$validated_bot && $claims_to_be_human && !$has_cookies) {
            $content =
                '<!DOCTYPE html>' .
                '<html lang="en">' .
                '<head>' .
                '<meta charset="utf-8">' .
                '<title>Cookie check</title>' .
                '<meta http-equiv="refresh" content="0">' .
                '</head>' .
                '<body>Cookie check</body>' .
                '</html>';

            return $this->response($content)
                ->withHeader('set-cookie', 'x=y; HttpOnly; SameSite=Strict');
        }

        // Bots get restricted access
        if ($validated_bot || $suspected_bot) {
            $request = $request->withAttribute(self::ROBOT_ATTRIBUTE_NAME, true);
        }

        // Scans for WordPress vulnerabilities?
        // Block these before wasting resources on DB connections, sessions, etc.
        $path = $request->getUri()->getPath();

        if (str_starts_with($path, '/xmlrpc.php') || str_starts_with($path, '/wp-')) {
            return $this->response('Not acceptable: not-wp');
        }

        return $handler->handle($request);
    }

    /**
     * Check that an IP address belongs to a robot operator using a forward/reverse DNS lookup.
     *
     * @param list<string> $valid_domains
     */
    private function checkRobotDNS(string $ip, array $valid_domains, bool $reverse_only): bool
    {
        $host = gethostbyaddr($ip);

        if ($host === false) {
            return false;
        }

        foreach ($valid_domains as $domain) {
            if (str_ends_with($host, $domain)) {
                return $reverse_only || $ip === gethostbyname($host);
            }
        }

        return false;
    }

    /**
     * @return array<RangeInterface>
     */
    private function fetchIpRangesForAsn(string $asn): array
    {
        return Registry::cache()->file()->remember('whois-asn-' . $asn, function () use ($asn): array {
            $ranges = $this->network_service->findIpRangesForAsn($asn);
            $mapper = static fn (string $range): ?RangeInterface => Factory::parseRangeString($range);
            $ranges = array_map($mapper, $ranges);

            return array_filter($ranges);
        }, random_int(self::WHOIS_TTL_MIN, self::WHOIS_TTL_MAX));
    }

    private function response(string $content): ResponseInterface
    {
        return response($content, StatusCodeInterface::STATUS_NOT_ACCEPTABLE);
    }
}
