From 0922feaea2c63290ced00a76b6c5046e21f5226f Mon Sep 17 00:00:00 2001 From: Robert Wachs Date: Wed, 3 Jul 2024 11:21:39 +0200 Subject: [PATCH 1/2] detect Apache HTTPClient as bot --- lib/detection_rules_bots.yml | 8 ++++++++ lib/pod_ident/detection_rules_bots.rb | 2 +- spec/detection_rules_bots.rb | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/detection_rules_bots.yml b/lib/detection_rules_bots.yml index a7b62a9..83d08a9 100644 --- a/lib/detection_rules_bots.yml +++ b/lib/detection_rules_bots.yml @@ -51,6 +51,14 @@ test: userAgents: - userAgent: AnchorImport/1.0 +- app: Apache HTTPClient + match: + regex: "^Apache\\-HttpClient" + platform: bot + test: + userAgents: + - userAgent: Apache-HttpClient/5.1.4 (Java/1.8.0_144) + - userAgent: Apache-HttpClient/5.1.4 (Java/1.8.0_91) - app: Apple Podcasts (Watch) match: regex: "^atc/|\\(null\\) watchOS/" diff --git a/lib/pod_ident/detection_rules_bots.rb b/lib/pod_ident/detection_rules_bots.rb index 0539d8f..88d67cc 100644 --- a/lib/pod_ident/detection_rules_bots.rb +++ b/lib/pod_ident/detection_rules_bots.rb @@ -1,3 +1,3 @@ # DO NOT EDIT THIS FILE - it gets automatically generated by running "bin/parse-rules" -BOTS_RULES = [{:app=>"AAABot - unknown bot", :match=>{"regex"=>"AAABot"}, :platform=>"bot"}, {:app=>"Adswizz", :match=>{"regex"=>"^AIS VirtualListener"}, :platform=>"bot"}, {:app=>"AhrefsBot", :match=>{"regex"=>"AhrefsBot/"}, :platform=>"bot"}, {:app=>"AirableBot", :match=>{"regex"=>"AirableBot-Podcast/"}, :platform=>"bot"}, {:app=>"Alexa Flash Briefing cache", :match=>{"regex"=>"^AmazonNewsContentService"}, :platform=>"bot"}, {:app=>"AlignaBot", :match=>{"regex"=>"^Alignabot"}, :platform=>"bot"}, {:app=>"Amazon Music Podcasts Bot", :match=>{"regex"=>"^Amazon Music Podcast"}, :platform=>"bot"}, {:app=>"Anchor Importer", :match=>{"regex"=>"AnchorImport"}, :platform=>"bot"}, {:app=>"Apple Podcasts (Watch)", :match=>{"regex"=>"^atc/|\\(null\\) watchOS/"}, :platform=>"bot"}, {:app=>"Apple Podcasts automated checks", :match=>{"regex"=>"(iTMS|itunesstored|itms)"}, :platform=>"bot"}, {:app=>"Applebot", :match=>{"regex"=>"^Applebot/"}, :platform=>"bot"}, {:app=>"Archive.org", :match=>{"regex"=>"archive\\.org_bot"}, :platform=>"bot"}, {:app=>"atheerfm", :match=>{"regex"=>"^atheerfm/"}, :platform=>"bot"}, {:app=>"Audiomack", :match=>{"regex"=>"^Audiomack Podcast Processor/"}, :platform=>"bot"}, {:app=>"AudioWave feed parser", :match=>{"regex"=>"^AudioWaveBot/1\\.0"}, :platform=>"bot"}, {:app=>"AwarioSmartBot", :match=>{"regex"=>"^AwarioSmartBot/"}, :platform=>"bot"}, {:app=>"Babbar", :match=>{"regex"=>"Barkrowler/"}, :platform=>"bot"}, {:app=>"Baidu", :match=>{"regex"=>"\\(ce\\.baidu\\.com"}, :platform=>"bot"}, {:app=>"bbot", :match=>{"regex"=>"^bbot/"}, :platform=>"bot"}, {:app=>"British Library", :match=>{"regex"=>"^bl\\.uk_ldfc_bot"}, :platform=>"bot"}, {:app=>"Blubrry Migration Service", :match=>{"regex"=>"^Blubrry Migration Service"}, :platform=>"bot"}, {:app=>"Buzzsprout Importer", :match=>{"regex"=>"^Buzzsprout Importer"}, :platform=>"bot"}, {:app=>"CastFeedValidator", :match=>{"regex"=>"^CastFeedValidator/"}, :platform=>"bot"}, {:app=>"Castopod", :match=>{"regex"=>"Castopod/\\d"}, :platform=>"bot"}, {:app=>"Chartable", :match=>{"regex"=>"^Trackable/"}, :platform=>"bot"}, {:app=>"Clark-Crawler, unknown", :match=>{"regex"=>"^clark-crawler2"}, :platform=>"bot"}, {:app=>"Critical Mention", :match=>{"regex"=>"^Podcast-CriticalMention/"}, :platform=>"bot"}, {:app=>"curl", :match=>{"regex"=>"^curl|^libcurl/|^PycURL/| curl/"}, :platform=>"bot"}, {:app=>"DataforSEO", :match=>{"regex"=>"DataForSeoBot/"}, :platform=>"bot"}, {:app=>"Datagnion Bot", :match=>{"regex"=>"^datagnionbot"}, :platform=>"bot"}, {:app=>"dataprovider.com", :match=>{"regex"=>"Dataprovider\\.com"}, :platform=>"bot"}, {:app=>"Daum", :match=>{"regex"=>"http://cs\\.daum\\.net"}, :platform=>"bot"}, {:app=>"Deezer Podcasters", :match=>{"regex"=>"^Deezer Podcasters/1\\.0"}, :platform=>"bot"}, {:app=>"Digg", :match=>{"regex"=>"^Digg "}, :platform=>"bot"}, {:app=>"dorada", :match=>{"regex"=>"support@dorada\\.co\\.uk"}, :platform=>"bot"}, {:app=>"DotBot", :match=>{"regex"=>"DotBot"}, :platform=>"bot"}, {:app=>"Downcast Bot", :match=>{"regex"=>"downcast feed consumer/"}, :platform=>"bot"}, {:app=>"DuckDuckBot", :match=>{"regex"=>"DuckDuckBot"}, :platform=>"bot"}, {:app=>"F-Secure Riddler", :match=>{"regex"=>"^Riddler "}, :platform=>"bot"}, {:app=>"Facebook Bot", :match=>{"regex"=>"FacebookBot|f?acebookexternalhit/?|^podcastbot$|Facebot|facebookexternalua|^facebookplatform/"}, :platform=>"bot"}, {:app=>"Feedly", :match=>{"regex"=>"^Feedly/"}, :platform=>"bot"}, {:app=>"Feedspot Fetcher", :match=>{"regex"=>"https://www\\.feedspot\\.com/fs/fetcher"}, :platform=>"bot"}, {:app=>"FlexGet", :match=>{"regex"=>"^FlexGet/"}, :platform=>"bot"}, {:app=>"Flipboard Proxy", :match=>{"regex"=>"FlipboardProxy/"}, :platform=>"bot"}, {:app=>"Fyyd", :match=>{"regex"=>"^fyyd-poll"}, :platform=>"bot"}, {:app=>"Go Storage Gateway V1", :match=>{"regex"=>"^storagegw-v1-go$"}, :platform=>"bot"}, {:app=>"Goodpods Bot", :match=>{"regex"=>"Goodpods/\\d+\\.\\d+"}, :platform=>"bot"}, {:app=>"Google Adsense Bot", :match=>{"regex"=>"Mediapartners-Google"}, :platform=>"bot"}, {:app=>"Google AdsBot", :match=>{"regex"=>"AdsBot-Google"}, :platform=>"bot"}, {:app=>"Google Podcasts Manager", :match=>{"regex"=>"Google-Podcast"}, :platform=>"bot"}, {:app=>"Googlebot", :match=>{"regex"=>"Googlebot/|Googlebot-Video/|Googlebot-Image/|^Feedfetcher-Google|google-xrawler"}, :platform=>"bot"}, {:app=>"Gumball", :match=>{"regex"=>"^Gumball"}, :platform=>"bot"}, {:app=>"Headliner", :match=>{"regex"=>"Headliner/"}, :platform=>"bot"}, {:app=>"HubSpot Crawler", :match=>{"regex"=>"HubSpot Crawler"}, :platform=>"bot"}, {:app=>"Internet Archive", :match=>{"regex"=>"Archive-It;|web\\.archive\\.org"}, :platform=>"bot"}, {:app=>"Jaunt", :match=>{"regex"=>"^Jaunt/"}, :platform=>"bot"}, {:app=>"l'Institut national de l'audiovisuel", :match=>{"regex"=>"INA dlweb"}, :platform=>"bot"}, {:app=>"Libsyn", :match=>{"regex"=>"^Libsyn4"}, :platform=>"bot"}, {:app=>"libwww-perl", :match=>{"regex"=>"^libwww-perl| libwww-perl"}, :platform=>"bot"}, {:app=>"Livelap Crawler", :match=>{"regex"=>"LivelapBot"}, :platform=>"bot"}, {:app=>"LTX71", :match=>{"regex"=>"^ltx71 "}, :platform=>"bot"}, {:app=>"MauiBot", :match=>{"regex"=>"^MauiBot"}, :platform=>"bot"}, {:app=>"Mastodon Bot", :match=>{"regex"=>"rb/.*Mastodon/"}, :platform=>"bot"}, {:app=>"Microsoft Bingbot", :match=>{"regex"=>"(BingPreview/|adidxbot/|[bB]ingbot/)"}, :platform=>"bot"}, {:app=>"Microsoft Office", :match=>{"regex"=>"ms-office; MSOffice"}, :platform=>"bot"}, {:app=>"Mixcloud Podcast Importer", :match=>{"regex"=>"^MixcloudPodcastImporter/"}, :platform=>"bot"}, {:app=>"MJ12bot", :match=>{"regex"=>".*MJ12bot"}, :platform=>"bot"}, {:app=>"Mozilla Bot", :match=>{"regex"=>"^'?Mozilla(/5\\.0(\\.\\.\\.)?)?$|^\\(Mozilla/5\\.0\\)$"}, :platform=>"bot"}, {:app=>"MSN Bot", :match=>{"regex"=>"^msnbot/"}, :platform=>"bot"}, {:app=>"Neevabot", :match=>{"regex"=>".*Neevabot"}, :platform=>"bot"}, {:app=>"Netcraft Survey Agent", :match=>{"regex"=>" NetcraftSurveyAgent/"}, :platform=>"bot"}, {:app=>"Newspaper", :match=>{"regex"=>"^newspaper/\\d"}, :platform=>"bot"}, {:app=>"OgScrper", :match=>{"regex"=>"OgScrper"}, :platform=>"bot"}, {:app=>"OkDownload", :match=>{"regex"=>"^OkDownload/"}, :platform=>"bot"}, {:app=>"OP3 Fetcher", :match=>{"regex"=>"^op3-fetcher/"}, :platform=>"bot"}, {:app=>"Overcast feed parser", :match=>{"regex"=>"^Overcast/1\\.0 Podcast Sync"}, :platform=>"bot"}, {:app=>"Pandora RSS crawler", :match=>{"regex"=>"^PandoraRSSCrawler"}, :platform=>"bot"}, {:app=>"PaperLi", :match=>{"regex"=>"PaperLiBot/"}, :platform=>"bot"}, {:app=>"PetalBot", :match=>{"regex"=>"PetalBot"}, :platform=>"bot"}, {:app=>"Pingdom", :match=>{"regex"=>"^Pingdom"}, :platform=>"bot"}, {:app=>"PlayerFM Podcast Sync", :match=>{"regex"=>"PlayerFM/.* Podcast Sync"}, :platform=>"bot"}, {:app=>"Podbean Importer", :match=>{"regex"=>"^Podbean Importer"}, :platform=>"bot"}, {:app=>"Podcastindex.org", :match=>{"regex"=>"^Podcastindex\\.org/"}, :platform=>"bot"}, {:app=>"Podcast de facto Standard", :match=>{"regex"=>"^PodcastStandard/"}, :platform=>"bot"}, {:app=>"Podcast Archiver", :match=>{"regex"=>"^Podcast%20Archiver/"}, :platform=>"bot"}, {:app=>"PodCloud", :match=>{"regex"=>"podCloud"}, :platform=>"bot"}, {:app=>"Podcorn", :match=>{"regex"=>"Podcorn/"}, :platform=>"bot"}, {:app=>"PodderBot", :match=>{"regex"=>"PodderBot/"}, :platform=>"bot"}, {:app=>"Podfollow", :match=>{"regex"=>"podfollowbot/"}, :platform=>"bot"}, {:app=>"Podgrab", :match=>{"regex"=>"^Podgrab$"}, :platform=>"bot"}, {:app=>"Podhound", :match=>{"regex"=>"PodhoundBeta"}, :platform=>"bot"}, {:app=>"Podio Bot", :match=>{"regex"=>"^Podio/"}, :platform=>"bot"}, {:app=>"Podnews", :match=>{"regex"=>"PodnewsBot"}, :platform=>"bot"}, {:app=>"Podnods Bot", :match=>{"regex"=>"(podnods-crawler|podnods)"}, :platform=>"bot"}, {:app=>"Podscribe", :match=>{"regex"=>"(^Adswizz-podscribe/|^Podscribe/)"}, :platform=>"bot"}, {:app=>"Podverse Feed Parser", :match=>{"regex"=>"^Podverse/Feed Parser"}, :platform=>"bot"}, {:app=>"PodvineBot", :match=>{"regex"=>"^PodvineBot/"}, :platform=>"bot"}, {:app=>"PostRank Bot", :match=>{"regex"=>"^PostRank/"}, :platform=>"bot"}, {:app=>"PodwatchPro", :match=>{"regex"=>"Podwatch-Pro Crawler"}, :platform=>"bot"}, {:app=>"Puppeteer", :match=>{"regex"=>" HeadlessChrome/\\d"}, :platform=>"bot"}, {:app=>"python-requests", :match=>{"regex"=>"python-requests"}, :platform=>"bot"}, {:app=>"Qiniu spider", :match=>{"regex"=>"qiniu.*spider"}, :platform=>"bot"}, {:app=>"RedCircle", :match=>{"regex"=>"RedCircle"}, :platform=>"bot"}, {:app=>"Ridder Bot", :match=>{"regex"=>" RidderBot/"}, :platform=>"bot"}, {:app=>"RSS to Telegram", :match=>{"regex"=>"^RSStT"}, :platform=>"bot"}, {:app=>"RSSRadio", :match=>{"regex"=>"^RSSRadio \\("}, :platform=>"bot"}, {:app=>"Ruby Mechanize", :match=>{"regex"=>"^Mechanize|[ -]Mechanize/"}, :platform=>"bot"}, {:app=>"Screaming Frog SEO Spider", :match=>{"regex"=>"^Screaming Frog "}, :platform=>"bot"}, {:app=>"SearchAtlas.com SEO Crawler", :match=>{"regex"=>"^SearchAtlas.*Crawler"}, :platform=>"bot"}, {:app=>"SEMrushBot", :match=>{"regex"=>"SemrushBot/|^SEMrushBot$"}, :platform=>"bot"}, {:app=>"Serendeputy", :match=>{"regex"=>"SerendeputyBot/"}, :platform=>"bot"}, {:app=>"Simplecast", :match=>{"regex"=>"^Simplecast$"}, :platform=>"bot"}, {:app=>"Slack Bot", :match=>{"regex"=>"^Slackbot 1\\.0"}, :platform=>"bot"}, {:app=>"Snapchat Bot", :match=>{"regex"=>"://developers\\.snap\\.com/robots"}, :platform=>"bot"}, {:app=>"SoundOn Bot", :match=>{"regex"=>"^SoundOn/[\\d.]+\\s+\\(bot"}, :platform=>"bot"}, {:app=>"Spotify cache service", :match=>{"regex"=>"^Spotify/1\\.0$"}, :platform=>"bot"}, {:app=>"Stitcher Bot", :match=>{"regex"=>"^StitcherBot"}, :platform=>"bot"}, {:app=>"Substack Content Fetcher", :match=>{"regex"=>"^SubstackContentFetch/"}, :platform=>"bot"}, {:app=>"SupportingCast", :match=>{"regex"=>"^SupportingCast(/.*)?$"}, :platform=>"bot"}, {:app=>"Timpi search crawler", :match=>{"regex"=>"Timpibot/"}, :platform=>"bot"}, {:app=>"Tiny Tiny RSS", :match=>{"regex"=>"^Tiny Tiny RSS/"}, :platform=>"bot"}, {:app=>"Trendsmap Resolver", :match=>{"regex"=>"TrendsmapResolver/"}, :platform=>"bot"}, {:app=>"Twingly Bot", :match=>{"regex"=>"Twingly Recon;"}, :platform=>"bot"}, {:app=>"Twitterbot", :match=>{"regex"=>"^Twitterbot"}, :platform=>"bot"}, {:app=>"Typhoeus", :match=>{"regex"=>"^Typhoeus"}, :platform=>"bot"}, {:app=>"UCast", :match=>{"regex"=>"^UCast/"}, :platform=>"bot"}, {:app=>"TelegramBot", :match=>{"regex"=>"^TelegramBot "}, :platform=>"bot"}, {:app=>"theTradeDesk Content Web Scraper", :match=>{"regex"=>"TTD-Content"}, :platform=>"bot"}, {:app=>"Veritone Engine Toolkit", :match=>{"regex"=>"^veritone/engine-toolkit"}, :platform=>"bot"}, {:app=>"Vurbl", :match=>{"regex"=>"VurblBot"}, :platform=>"bot"}, {:app=>"Wget", :match=>{"regex"=>"Wget"}, :platform=>"bot"}, {:app=>"weborama", :match=>{"regex"=>"^weborama-fetcher"}, :platform=>"bot"}, {:app=>"Windows Crawler", :match=>{"regex"=>"^ZDM/.*Windows"}, :platform=>"bot"}, {:app=>"Windows PowerShell", :match=>{"regex"=>"WindowsPowerShell/"}, :platform=>"bot"}, {:app=>"WordPress", :match=>{"regex"=>"^WordPress"}, :platform=>"bot"}, {:app=>"wsrv.nl", :match=>{"regex"=>"https?://wsrv.nl/"}, :platform=>"bot"}, {:app=>"YaCy", :match=>{"regex"=>"^yacybot"}, :platform=>"bot"}, {:app=>"Yahoo Crawler", :match=>{"regex"=>"Yahoo! Slurp"}, :platform=>"bot"}, {:app=>"YandexBot", :match=>{"regex"=>"YandexBot/"}, :platform=>"bot"}, {:app=>"Zapier", :match=>{"regex"=>"^Zapier$"}, :platform=>"bot"}, {:app=>"Zencast", :match=>{"regex"=>"^Zencastr/"}, :platform=>"bot"}].freeze \ No newline at end of file +BOTS_RULES = [{:app=>"AAABot - unknown bot", :match=>{"regex"=>"AAABot"}, :platform=>"bot"}, {:app=>"Adswizz", :match=>{"regex"=>"^AIS VirtualListener"}, :platform=>"bot"}, {:app=>"AhrefsBot", :match=>{"regex"=>"AhrefsBot/"}, :platform=>"bot"}, {:app=>"AirableBot", :match=>{"regex"=>"AirableBot-Podcast/"}, :platform=>"bot"}, {:app=>"Alexa Flash Briefing cache", :match=>{"regex"=>"^AmazonNewsContentService"}, :platform=>"bot"}, {:app=>"AlignaBot", :match=>{"regex"=>"^Alignabot"}, :platform=>"bot"}, {:app=>"Amazon Music Podcasts Bot", :match=>{"regex"=>"^Amazon Music Podcast"}, :platform=>"bot"}, {:app=>"Anchor Importer", :match=>{"regex"=>"AnchorImport"}, :platform=>"bot"}, {:app=>"Apache HTTPClient", :match=>{"regex"=>"^Apache\\-HttpClient"}, :platform=>"bot"}, {:app=>"Apple Podcasts (Watch)", :match=>{"regex"=>"^atc/|\\(null\\) watchOS/"}, :platform=>"bot"}, {:app=>"Apple Podcasts automated checks", :match=>{"regex"=>"(iTMS|itunesstored|itms)"}, :platform=>"bot"}, {:app=>"Applebot", :match=>{"regex"=>"^Applebot/"}, :platform=>"bot"}, {:app=>"Archive.org", :match=>{"regex"=>"archive\\.org_bot"}, :platform=>"bot"}, {:app=>"atheerfm", :match=>{"regex"=>"^atheerfm/"}, :platform=>"bot"}, {:app=>"Audiomack", :match=>{"regex"=>"^Audiomack Podcast Processor/"}, :platform=>"bot"}, {:app=>"AudioWave feed parser", :match=>{"regex"=>"^AudioWaveBot/1\\.0"}, :platform=>"bot"}, {:app=>"AwarioSmartBot", :match=>{"regex"=>"^AwarioSmartBot/"}, :platform=>"bot"}, {:app=>"Babbar", :match=>{"regex"=>"Barkrowler/"}, :platform=>"bot"}, {:app=>"Baidu", :match=>{"regex"=>"\\(ce\\.baidu\\.com"}, :platform=>"bot"}, {:app=>"bbot", :match=>{"regex"=>"^bbot/"}, :platform=>"bot"}, {:app=>"British Library", :match=>{"regex"=>"^bl\\.uk_ldfc_bot"}, :platform=>"bot"}, {:app=>"Blubrry Migration Service", :match=>{"regex"=>"^Blubrry Migration Service"}, :platform=>"bot"}, {:app=>"Buzzsprout Importer", :match=>{"regex"=>"^Buzzsprout Importer"}, :platform=>"bot"}, {:app=>"CastFeedValidator", :match=>{"regex"=>"^CastFeedValidator/"}, :platform=>"bot"}, {:app=>"Castopod", :match=>{"regex"=>"Castopod/\\d"}, :platform=>"bot"}, {:app=>"Chartable", :match=>{"regex"=>"^Trackable/"}, :platform=>"bot"}, {:app=>"Clark-Crawler, unknown", :match=>{"regex"=>"^clark-crawler2"}, :platform=>"bot"}, {:app=>"Critical Mention", :match=>{"regex"=>"^Podcast-CriticalMention/"}, :platform=>"bot"}, {:app=>"curl", :match=>{"regex"=>"^curl|^libcurl/|^PycURL/| curl/"}, :platform=>"bot"}, {:app=>"DataforSEO", :match=>{"regex"=>"DataForSeoBot/"}, :platform=>"bot"}, {:app=>"Datagnion Bot", :match=>{"regex"=>"^datagnionbot"}, :platform=>"bot"}, {:app=>"dataprovider.com", :match=>{"regex"=>"Dataprovider\\.com"}, :platform=>"bot"}, {:app=>"Daum", :match=>{"regex"=>"http://cs\\.daum\\.net"}, :platform=>"bot"}, {:app=>"Deezer Podcasters", :match=>{"regex"=>"^Deezer Podcasters/1\\.0"}, :platform=>"bot"}, {:app=>"Digg", :match=>{"regex"=>"^Digg "}, :platform=>"bot"}, {:app=>"dorada", :match=>{"regex"=>"support@dorada\\.co\\.uk"}, :platform=>"bot"}, {:app=>"DotBot", :match=>{"regex"=>"DotBot"}, :platform=>"bot"}, {:app=>"Downcast Bot", :match=>{"regex"=>"downcast feed consumer/"}, :platform=>"bot"}, {:app=>"DuckDuckBot", :match=>{"regex"=>"DuckDuckBot"}, :platform=>"bot"}, {:app=>"F-Secure Riddler", :match=>{"regex"=>"^Riddler "}, :platform=>"bot"}, {:app=>"Facebook Bot", :match=>{"regex"=>"FacebookBot|f?acebookexternalhit/?|^podcastbot$|Facebot|facebookexternalua|^facebookplatform/"}, :platform=>"bot"}, {:app=>"Feedly", :match=>{"regex"=>"^Feedly/"}, :platform=>"bot"}, {:app=>"Feedspot Fetcher", :match=>{"regex"=>"https://www\\.feedspot\\.com/fs/fetcher"}, :platform=>"bot"}, {:app=>"FlexGet", :match=>{"regex"=>"^FlexGet/"}, :platform=>"bot"}, {:app=>"Flipboard Proxy", :match=>{"regex"=>"FlipboardProxy/"}, :platform=>"bot"}, {:app=>"Fyyd", :match=>{"regex"=>"^fyyd-poll"}, :platform=>"bot"}, {:app=>"Go Storage Gateway V1", :match=>{"regex"=>"^storagegw-v1-go$"}, :platform=>"bot"}, {:app=>"Goodpods Bot", :match=>{"regex"=>"Goodpods/\\d+\\.\\d+"}, :platform=>"bot"}, {:app=>"Google Adsense Bot", :match=>{"regex"=>"Mediapartners-Google"}, :platform=>"bot"}, {:app=>"Google AdsBot", :match=>{"regex"=>"AdsBot-Google"}, :platform=>"bot"}, {:app=>"Google Podcasts Manager", :match=>{"regex"=>"Google-Podcast"}, :platform=>"bot"}, {:app=>"Googlebot", :match=>{"regex"=>"Googlebot/|Googlebot-Video/|Googlebot-Image/|^Feedfetcher-Google|google-xrawler"}, :platform=>"bot"}, {:app=>"Gumball", :match=>{"regex"=>"^Gumball"}, :platform=>"bot"}, {:app=>"Headliner", :match=>{"regex"=>"Headliner/"}, :platform=>"bot"}, {:app=>"HubSpot Crawler", :match=>{"regex"=>"HubSpot Crawler"}, :platform=>"bot"}, {:app=>"Internet Archive", :match=>{"regex"=>"Archive-It;|web\\.archive\\.org"}, :platform=>"bot"}, {:app=>"Jaunt", :match=>{"regex"=>"^Jaunt/"}, :platform=>"bot"}, {:app=>"l'Institut national de l'audiovisuel", :match=>{"regex"=>"INA dlweb"}, :platform=>"bot"}, {:app=>"Libsyn", :match=>{"regex"=>"^Libsyn4"}, :platform=>"bot"}, {:app=>"libwww-perl", :match=>{"regex"=>"^libwww-perl| libwww-perl"}, :platform=>"bot"}, {:app=>"Livelap Crawler", :match=>{"regex"=>"LivelapBot"}, :platform=>"bot"}, {:app=>"LTX71", :match=>{"regex"=>"^ltx71 "}, :platform=>"bot"}, {:app=>"MauiBot", :match=>{"regex"=>"^MauiBot"}, :platform=>"bot"}, {:app=>"Mastodon Bot", :match=>{"regex"=>"rb/.*Mastodon/"}, :platform=>"bot"}, {:app=>"Microsoft Bingbot", :match=>{"regex"=>"(BingPreview/|adidxbot/|[bB]ingbot/)"}, :platform=>"bot"}, {:app=>"Microsoft Office", :match=>{"regex"=>"ms-office; MSOffice"}, :platform=>"bot"}, {:app=>"Mixcloud Podcast Importer", :match=>{"regex"=>"^MixcloudPodcastImporter/"}, :platform=>"bot"}, {:app=>"MJ12bot", :match=>{"regex"=>".*MJ12bot"}, :platform=>"bot"}, {:app=>"Mozilla Bot", :match=>{"regex"=>"^'?Mozilla(/5\\.0(\\.\\.\\.)?)?$|^\\(Mozilla/5\\.0\\)$"}, :platform=>"bot"}, {:app=>"MSN Bot", :match=>{"regex"=>"^msnbot/"}, :platform=>"bot"}, {:app=>"Neevabot", :match=>{"regex"=>".*Neevabot"}, :platform=>"bot"}, {:app=>"Netcraft Survey Agent", :match=>{"regex"=>" NetcraftSurveyAgent/"}, :platform=>"bot"}, {:app=>"Newspaper", :match=>{"regex"=>"^newspaper/\\d"}, :platform=>"bot"}, {:app=>"OgScrper", :match=>{"regex"=>"OgScrper"}, :platform=>"bot"}, {:app=>"OkDownload", :match=>{"regex"=>"^OkDownload/"}, :platform=>"bot"}, {:app=>"OP3 Fetcher", :match=>{"regex"=>"^op3-fetcher/"}, :platform=>"bot"}, {:app=>"Overcast feed parser", :match=>{"regex"=>"^Overcast/1\\.0 Podcast Sync"}, :platform=>"bot"}, {:app=>"Pandora RSS crawler", :match=>{"regex"=>"^PandoraRSSCrawler"}, :platform=>"bot"}, {:app=>"PaperLi", :match=>{"regex"=>"PaperLiBot/"}, :platform=>"bot"}, {:app=>"PetalBot", :match=>{"regex"=>"PetalBot"}, :platform=>"bot"}, {:app=>"Pingdom", :match=>{"regex"=>"^Pingdom"}, :platform=>"bot"}, {:app=>"PlayerFM Podcast Sync", :match=>{"regex"=>"PlayerFM/.* Podcast Sync"}, :platform=>"bot"}, {:app=>"Podbean Importer", :match=>{"regex"=>"^Podbean Importer"}, :platform=>"bot"}, {:app=>"Podcastindex.org", :match=>{"regex"=>"^Podcastindex\\.org/"}, :platform=>"bot"}, {:app=>"Podcast de facto Standard", :match=>{"regex"=>"^PodcastStandard/"}, :platform=>"bot"}, {:app=>"Podcast Archiver", :match=>{"regex"=>"^Podcast%20Archiver/"}, :platform=>"bot"}, {:app=>"PodCloud", :match=>{"regex"=>"podCloud"}, :platform=>"bot"}, {:app=>"Podcorn", :match=>{"regex"=>"Podcorn/"}, :platform=>"bot"}, {:app=>"PodderBot", :match=>{"regex"=>"PodderBot/"}, :platform=>"bot"}, {:app=>"Podfollow", :match=>{"regex"=>"podfollowbot/"}, :platform=>"bot"}, {:app=>"Podgrab", :match=>{"regex"=>"^Podgrab$"}, :platform=>"bot"}, {:app=>"Podhound", :match=>{"regex"=>"PodhoundBeta"}, :platform=>"bot"}, {:app=>"Podio Bot", :match=>{"regex"=>"^Podio/"}, :platform=>"bot"}, {:app=>"Podnews", :match=>{"regex"=>"PodnewsBot"}, :platform=>"bot"}, {:app=>"Podnods Bot", :match=>{"regex"=>"(podnods-crawler|podnods)"}, :platform=>"bot"}, {:app=>"Podscribe", :match=>{"regex"=>"(^Adswizz-podscribe/|^Podscribe/)"}, :platform=>"bot"}, {:app=>"Podverse Feed Parser", :match=>{"regex"=>"^Podverse/Feed Parser"}, :platform=>"bot"}, {:app=>"PodvineBot", :match=>{"regex"=>"^PodvineBot/"}, :platform=>"bot"}, {:app=>"PostRank Bot", :match=>{"regex"=>"^PostRank/"}, :platform=>"bot"}, {:app=>"PodwatchPro", :match=>{"regex"=>"Podwatch-Pro Crawler"}, :platform=>"bot"}, {:app=>"Puppeteer", :match=>{"regex"=>" HeadlessChrome/\\d"}, :platform=>"bot"}, {:app=>"python-requests", :match=>{"regex"=>"python-requests"}, :platform=>"bot"}, {:app=>"Qiniu spider", :match=>{"regex"=>"qiniu.*spider"}, :platform=>"bot"}, {:app=>"RedCircle", :match=>{"regex"=>"RedCircle"}, :platform=>"bot"}, {:app=>"Ridder Bot", :match=>{"regex"=>" RidderBot/"}, :platform=>"bot"}, {:app=>"RSS to Telegram", :match=>{"regex"=>"^RSStT"}, :platform=>"bot"}, {:app=>"RSSRadio", :match=>{"regex"=>"^RSSRadio \\("}, :platform=>"bot"}, {:app=>"Ruby Mechanize", :match=>{"regex"=>"^Mechanize|[ -]Mechanize/"}, :platform=>"bot"}, {:app=>"Screaming Frog SEO Spider", :match=>{"regex"=>"^Screaming Frog "}, :platform=>"bot"}, {:app=>"SearchAtlas.com SEO Crawler", :match=>{"regex"=>"^SearchAtlas.*Crawler"}, :platform=>"bot"}, {:app=>"SEMrushBot", :match=>{"regex"=>"SemrushBot/|^SEMrushBot$"}, :platform=>"bot"}, {:app=>"Serendeputy", :match=>{"regex"=>"SerendeputyBot/"}, :platform=>"bot"}, {:app=>"Simplecast", :match=>{"regex"=>"^Simplecast$"}, :platform=>"bot"}, {:app=>"Slack Bot", :match=>{"regex"=>"^Slackbot 1\\.0"}, :platform=>"bot"}, {:app=>"Snapchat Bot", :match=>{"regex"=>"://developers\\.snap\\.com/robots"}, :platform=>"bot"}, {:app=>"SoundOn Bot", :match=>{"regex"=>"^SoundOn/[\\d.]+\\s+\\(bot"}, :platform=>"bot"}, {:app=>"Spotify cache service", :match=>{"regex"=>"^Spotify/1\\.0$"}, :platform=>"bot"}, {:app=>"Stitcher Bot", :match=>{"regex"=>"^StitcherBot"}, :platform=>"bot"}, {:app=>"Substack Content Fetcher", :match=>{"regex"=>"^SubstackContentFetch/"}, :platform=>"bot"}, {:app=>"SupportingCast", :match=>{"regex"=>"^SupportingCast(/.*)?$"}, :platform=>"bot"}, {:app=>"Timpi search crawler", :match=>{"regex"=>"Timpibot/"}, :platform=>"bot"}, {:app=>"Tiny Tiny RSS", :match=>{"regex"=>"^Tiny Tiny RSS/"}, :platform=>"bot"}, {:app=>"Trendsmap Resolver", :match=>{"regex"=>"TrendsmapResolver/"}, :platform=>"bot"}, {:app=>"Twingly Bot", :match=>{"regex"=>"Twingly Recon;"}, :platform=>"bot"}, {:app=>"Twitterbot", :match=>{"regex"=>"^Twitterbot"}, :platform=>"bot"}, {:app=>"Typhoeus", :match=>{"regex"=>"^Typhoeus"}, :platform=>"bot"}, {:app=>"UCast", :match=>{"regex"=>"^UCast/"}, :platform=>"bot"}, {:app=>"TelegramBot", :match=>{"regex"=>"^TelegramBot "}, :platform=>"bot"}, {:app=>"theTradeDesk Content Web Scraper", :match=>{"regex"=>"TTD-Content"}, :platform=>"bot"}, {:app=>"Veritone Engine Toolkit", :match=>{"regex"=>"^veritone/engine-toolkit"}, :platform=>"bot"}, {:app=>"Vurbl", :match=>{"regex"=>"VurblBot"}, :platform=>"bot"}, {:app=>"Wget", :match=>{"regex"=>"Wget"}, :platform=>"bot"}, {:app=>"weborama", :match=>{"regex"=>"^weborama-fetcher"}, :platform=>"bot"}, {:app=>"Windows Crawler", :match=>{"regex"=>"^ZDM/.*Windows"}, :platform=>"bot"}, {:app=>"Windows PowerShell", :match=>{"regex"=>"WindowsPowerShell/"}, :platform=>"bot"}, {:app=>"WordPress", :match=>{"regex"=>"^WordPress"}, :platform=>"bot"}, {:app=>"wsrv.nl", :match=>{"regex"=>"https?://wsrv.nl/"}, :platform=>"bot"}, {:app=>"YaCy", :match=>{"regex"=>"^yacybot"}, :platform=>"bot"}, {:app=>"Yahoo Crawler", :match=>{"regex"=>"Yahoo! Slurp"}, :platform=>"bot"}, {:app=>"YandexBot", :match=>{"regex"=>"YandexBot/"}, :platform=>"bot"}, {:app=>"Zapier", :match=>{"regex"=>"^Zapier$"}, :platform=>"bot"}, {:app=>"Zencast", :match=>{"regex"=>"^Zencastr/"}, :platform=>"bot"}].freeze \ No newline at end of file diff --git a/spec/detection_rules_bots.rb b/spec/detection_rules_bots.rb index 01848b6..505cc0a 100644 --- a/spec/detection_rules_bots.rb +++ b/spec/detection_rules_bots.rb @@ -1,3 +1,3 @@ # DO NOT EDIT THIS FILE - it gets automatically generated by running "bin/parse-rules" -BOTS_RULES = [{:app=>"AAABot - unknown bot", :match=>{"regex"=>"AAABot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AAABot"}]}}, {:app=>"Adswizz", :match=>{"regex"=>"^AIS VirtualListener"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AIS VirtualListener"}]}}, {:app=>"AhrefsBot", :match=>{"regex"=>"AhrefsBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; AhrefsBot/7.0; http://ahrefs.com/robot/)"}, {"userAgent"=>"Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)"}]}}, {:app=>"AirableBot", :match=>{"regex"=>"AirableBot-Podcast/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AirableBot-Podcast/1.0 (+https//www.airablenow.com)"}, {"userAgent"=>"AirableBot-Podcast/1.0 ( https//www.airablenow.com)"}]}}, {:app=>"Alexa Flash Briefing cache", :match=>{"regex"=>"^AmazonNewsContentService"}, :platform=>"bot"}, {:app=>"AlignaBot", :match=>{"regex"=>"^Alignabot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Alignabot 1.0 (https://www.alignable.com/)"}]}}, {:app=>"Amazon Music Podcasts Bot", :match=>{"regex"=>"^Amazon Music Podcast"}, :platform=>"bot"}, {:app=>"Anchor Importer", :match=>{"regex"=>"AnchorImport"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AnchorImport/1.0"}]}}, {:app=>"Apple Podcasts (Watch)", :match=>{"regex"=>"^atc/|\\(null\\) watchOS/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"atc/1.0"}, {"userAgent"=>"atc/1.0 watchOS/6.2 model/Watch3,3 hwp/t8004 build/17T529 (6; dt:155)"}, {"userAgent"=>"atc/1.0 watchOS/6.2.8 model/Watch2,3 hwp/t8002 build/17U63 (6; dt:133)"}, {"userAgent"=>"atc/1.0 watchOS/6.2.8 model/Watch3,3 hwp/t8004 build/17U63 (6; dt:155)"}, {"userAgent"=>"atc/1.0 watchOS/6.2.8 model/Watch4,2 hwp/t8006 build/17U63 (6; dt:191)"}, {"userAgent"=>"atc/1.0 watchOS/7.0.2 model/Watch5,10 hwp/t8006 build/18R402 (6; dt:233)"}, {"userAgent"=>"atc/1.0 watchOS/7.0.2 model/Watch5,11 hwp/t8006 build/18R402 (6; dt:234)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch4,2 hwp/t8006 build/18R590 (6; dt:191)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch4,3 hwp/t8006 build/18R590 (6; dt:192)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch4,4 hwp/t8006 build/18R590 (6; dt:193)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch5,1 hwp/t8006 build/18R590 (6; dt:201)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch5,3 hwp/t8006 build/18R590 (6; dt:202)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch5,4 hwp/t8006 build/18R590 (6; dt:202)"}, {"userAgent"=>"(null)/(null) watchOS/5.0.1 model/Watch3,3 hwp/t8004 build/16R381 (6; dt:155)"}]}}, {:app=>"Apple Podcasts automated checks", :match=>{"regex"=>"(iTMS|itunesstored|itms)"}, :platform=>"bot"}, {:app=>"Applebot", :match=>{"regex"=>"^Applebot/"}, :platform=>"bot"}, {:app=>"Archive.org", :match=>{"regex"=>"archive\\.org_bot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; archive.org_bot http://archive.org/details/archive.org_bot)"}]}}, {:app=>"atheerfm", :match=>{"regex"=>"^atheerfm/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"atheerfm/1 CFNetwork/758.3.15 Darwin/15.3.0"}]}}, {:app=>"Audiomack", :match=>{"regex"=>"^Audiomack Podcast Processor/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Audiomack Podcast Processor/1.0 (https://audiomack.com/)"}]}}, {:app=>"AudioWave feed parser", :match=>{"regex"=>"^AudioWaveBot/1\\.0"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AudioWaveBot/1.0"}]}}, {:app=>"AwarioSmartBot", :match=>{"regex"=>"^AwarioSmartBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)"}]}}, {:app=>"Babbar", :match=>{"regex"=>"Barkrowler/"}, :platform=>"bot"}, {:app=>"Baidu", :match=>{"regex"=>"\\(ce\\.baidu\\.com"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Baidu-YunGuanCe-SLABot(ce.baidu.com)"}]}}, {:app=>"bbot", :match=>{"regex"=>"^bbot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"bbot/0.1"}]}}, {:app=>"British Library", :match=>{"regex"=>"^bl\\.uk_ldfc_bot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"bl.uk_ldfc_bot/3.4.0-20200518 ( http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)"}]}}, {:app=>"Blubrry Migration Service", :match=>{"regex"=>"^Blubrry Migration Service"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Blubrry Migration Service"}]}}, {:app=>"Buzzsprout Importer", :match=>{"regex"=>"^Buzzsprout Importer"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Buzzsprout Importer"}]}}, {:app=>"CastFeedValidator", :match=>{"regex"=>"^CastFeedValidator/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"CastFeedValidator/3.0.5 (https://castfeedvalidator.com)"}]}}, {:app=>"Castopod", :match=>{"regex"=>"Castopod/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Castopod/1.0"}, {"userAgent"=>"Castopod/1.1.2"}]}}, {:app=>"Chartable", :match=>{"regex"=>"^Trackable/"}, :platform=>"bot"}, {:app=>"Clark-Crawler, unknown", :match=>{"regex"=>"^clark-crawler2"}, :platform=>"bot"}, {:app=>"Critical Mention", :match=>{"regex"=>"^Podcast-CriticalMention/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcast-CriticalMention/1.0"}]}}, {:app=>"curl", :match=>{"regex"=>"^curl|^libcurl/|^PycURL/| curl/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"libcurl/7.65.1"}, {"userAgent"=>"PycURL/7.43.0 libcurl/7.47.0 GnuTLS/3.4.10 zlib/1.2.8 libidn/1.32 librtmp/2.3"}, {"userAgent"=>"GuzzleHttp/6.1.0 curl/7.50.1 PHP/7.0.13-0ubuntu0.16.10.1"}]}}, {:app=>"DataforSEO", :match=>{"regex"=>"DataForSeoBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)"}]}}, {:app=>"Datagnion Bot", :match=>{"regex"=>"^datagnionbot"}, :platform=>"bot"}, {:app=>"dataprovider.com", :match=>{"regex"=>"Dataprovider\\.com"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Dataprovider.com)"}]}}, {:app=>"Daum", :match=>{"regex"=>"http://cs\\.daum\\.net"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)"}]}}, {:app=>"Deezer Podcasters", :match=>{"regex"=>"^Deezer Podcasters/1\\.0"}, :platform=>"bot"}, {:app=>"Digg", :match=>{"regex"=>"^Digg "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Digg Mozilla/5.0 (Digg/1.0; support@digg.com)"}]}}, {:app=>"dorada", :match=>{"regex"=>"support@dorada\\.co\\.uk"}, :platform=>"bot"}, {:app=>"DotBot", :match=>{"regex"=>"DotBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)"}, {"userAgent"=>"Mozilla/5.0 (compatible; DotBot/1.2; https://opensiteexplorer.org/dotbot; help@moz.com)"}]}}, {:app=>"Downcast Bot", :match=>{"regex"=>"downcast feed consumer/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"downcast feed consumer/0.0.175; (mode=dev; id=u2NgjBSPM6; downcast.fm)"}]}}, {:app=>"DuckDuckBot", :match=>{"regex"=>"DuckDuckBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)"}]}}, {:app=>"F-Secure Riddler", :match=>{"regex"=>"^Riddler "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Riddler (http://riddler.io/about)"}]}}, {:app=>"Facebook Bot", :match=>{"regex"=>"FacebookBot|f?acebookexternalhit/?|^podcastbot$|Facebot|facebookexternalua|^facebookplatform/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"facebookexternalhit/1.1 ( http://www.facebook.com/externalhit_uatext.php)"}, {"userAgent"=>"podcastbot"}, {"userAgent"=>"facebookexternalhit"}, {"userAgent"=>"Facebot"}, {"userAgent"=>"facebookexternalua"}, {"userAgent"=>"facebookplatform/1.0 (+http://developers.facebook.com)"}, {"userAgent"=>"acebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)"}]}}, {:app=>"Feedly", :match=>{"regex"=>"^Feedly/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)"}]}}, {:app=>"Feedspot Fetcher", :match=>{"regex"=>"https://www\\.feedspot\\.com/fs/fetcher"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)"}]}}, {:app=>"FlexGet", :match=>{"regex"=>"^FlexGet/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"FlexGet/3.3.4 (www.flexget.com)"}]}}, {:app=>"Flipboard Proxy", :match=>{"regex"=>"FlipboardProxy/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)"}]}}, {:app=>"Fyyd", :match=>{"regex"=>"^fyyd-poll"}, :platform=>"bot"}, {:app=>"Go Storage Gateway V1", :match=>{"regex"=>"^storagegw-v1-go$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"storagegw-v1-go"}]}}, {:app=>"Goodpods Bot", :match=>{"regex"=>"Goodpods/\\d+\\.\\d+"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Goodpods/2.2"}]}}, {:app=>"Google Adsense Bot", :match=>{"regex"=>"Mediapartners-Google"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mediapartners-Google"}]}}, {:app=>"Google AdsBot", :match=>{"regex"=>"AdsBot-Google"}, :platform=>"bot"}, {:app=>"Google Podcasts Manager", :match=>{"regex"=>"Google-Podcast"}, :platform=>"bot"}, {:app=>"Googlebot", :match=>{"regex"=>"Googlebot/|Googlebot-Video/|Googlebot-Image/|^Feedfetcher-Google|google-xrawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; http://www.google.com/bot.html)"}, {"userAgent"=>"Googlebot-Image/1.0"}, {"userAgent"=>"Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=4653447469100830145)"}, {"userAgent"=>"google-xrawler"}]}}, {:app=>"Gumball", :match=>{"regex"=>"^Gumball"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Gumball.fm Analytics Prefix Checker"}]}}, {:app=>"Headliner", :match=>{"regex"=>"Headliner/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Headliner/1.0.0 +https://headliner.app"}]}}, {:app=>"HubSpot Crawler", :match=>{"regex"=>"HubSpot Crawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; HubSpot Crawler; +https://www.hubspot.com)"}]}}, {:app=>"Internet Archive", :match=>{"regex"=>"Archive-It;|web\\.archive\\.org"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; special_archiver; Archive-It; http://archive-it.org/files/site-owners-special.html)"}, {"userAgent"=>"ia_archiver-web.archive.org"}]}}, {:app=>"Jaunt", :match=>{"regex"=>"^Jaunt/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Jaunt/1.5"}]}}, {:app=>"l'Institut national de l'audiovisuel", :match=>{"regex"=>"INA dlweb"}, :platform=>"bot"}, {:app=>"Libsyn", :match=>{"regex"=>"^Libsyn4"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Libsyn4-Download"}]}}, {:app=>"libwww-perl", :match=>{"regex"=>"^libwww-perl| libwww-perl"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"LWP::Simple/6.34 libwww-perl/6.34"}]}}, {:app=>"Livelap Crawler", :match=>{"regex"=>"LivelapBot"}, :platform=>"bot"}, {:app=>"LTX71", :match=>{"regex"=>"^ltx71 "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"ltx71 - (http://ltx71.com/)"}]}}, {:app=>"MauiBot", :match=>{"regex"=>"^MauiBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"MauiBot (crawler.feedback dc@gmail.com)"}]}}, {:app=>"Mastodon Bot", :match=>{"regex"=>"rb/.*Mastodon/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"http.rb/5.0.4 (Mastodon/3.5.3; +https://mastodon.xyz/) Bot"}]}}, {:app=>"Microsoft Bingbot", :match=>{"regex"=>"(BingPreview/|adidxbot/|[bB]ingbot/)"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534 (KHTML, like Gecko) BingPreview/1.0b"}, {"userAgent"=>"Mozilla/5.0 (compatible; Bingbot/2.0; +http://www.bing.com/bingbot.htm)"}]}}, {:app=>"Microsoft Office", :match=>{"regex"=>"ms-office; MSOffice"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/4.0 (compatible; ms-office; MSOffice 16)"}]}}, {:app=>"Mixcloud Podcast Importer", :match=>{"regex"=>"^MixcloudPodcastImporter/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"MixcloudPodcastImporter/2.0"}]}}, {:app=>"MJ12bot", :match=>{"regex"=>".*MJ12bot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)"}]}}, {:app=>"Mozilla Bot", :match=>{"regex"=>"^'?Mozilla(/5\\.0(\\.\\.\\.)?)?$|^\\(Mozilla/5\\.0\\)$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0"}, {"userAgent"=>"Mozilla"}, {"userAgent"=>"Mozilla/5.0..."}, {"userAgent"=>"'Mozilla/5.0"}, {"userAgent"=>"(Mozilla/5.0)"}]}}, {:app=>"MSN Bot", :match=>{"regex"=>"^msnbot/"}, :platform=>"bot"}, {:app=>"Neevabot", :match=>{"regex"=>".*Neevabot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Neevabot/1.0; https://neeva.com/neevabot)"}]}}, {:app=>"Netcraft Survey Agent", :match=>{"regex"=>" NetcraftSurveyAgent/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)"}]}}, {:app=>"Newspaper", :match=>{"regex"=>"^newspaper/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"newspaper/0.3.0"}]}}, {:app=>"OgScrper", :match=>{"regex"=>"OgScrper"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"OgScrper"}]}}, {:app=>"OkDownload", :match=>{"regex"=>"^OkDownload/"}, :platform=>"bot"}, {:app=>"OP3 Fetcher", :match=>{"regex"=>"^op3-fetcher/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"op3-fetcher/1.0 (https://op3.dev)"}]}}, {:app=>"Overcast feed parser", :match=>{"regex"=>"^Overcast/1\\.0 Podcast Sync"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Overcast/1.0 Podcast Sync"}]}}, {:app=>"Pandora RSS crawler", :match=>{"regex"=>"^PandoraRSSCrawler"}, :platform=>"bot"}, {:app=>"PaperLi", :match=>{"regex"=>"PaperLiBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)"}]}}, {:app=>"PetalBot", :match=>{"regex"=>"PetalBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot; https://aspiegel.com/petalbot)"}]}}, {:app=>"Pingdom", :match=>{"regex"=>"^Pingdom"}, :platform=>"bot"}, {:app=>"PlayerFM Podcast Sync", :match=>{"regex"=>"PlayerFM/.* Podcast Sync"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PlayerFM/1.0 Podcast Sync (0 subscribers; url=https://player.fm/series/series-2567872)"}]}}, {:app=>"Podbean Importer", :match=>{"regex"=>"^Podbean Importer"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podbean Importer"}]}}, {:app=>"Podcastindex.org", :match=>{"regex"=>"^Podcastindex\\.org/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcastindex.org/v0.3.3 (Aggrivate)"}]}}, {:app=>"Podcast de facto Standard", :match=>{"regex"=>"^PodcastStandard/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodcastStandard/0.1.0 (+https://podcast-standard.org)"}]}}, {:app=>"Podcast Archiver", :match=>{"regex"=>"^Podcast%20Archiver/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcast%20Archiver/3000000003 CFNetwork/1406.0.4 Darwin/22.4.0"}]}}, {:app=>"PodCloud", :match=>{"regex"=>"podCloud"}, :platform=>"bot"}, {:app=>"Podcorn", :match=>{"regex"=>"Podcorn/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcorn/1.0"}]}}, {:app=>"PodderBot", :match=>{"regex"=>"PodderBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodderBot/1.0"}]}}, {:app=>"Podfollow", :match=>{"regex"=>"podfollowbot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 https://podfollow.com/crawling podfollowbot/1.0"}]}}, {:app=>"Podgrab", :match=>{"regex"=>"^Podgrab$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podgrab"}]}}, {:app=>"Podhound", :match=>{"regex"=>"PodhoundBeta"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodhoundBeta"}]}}, {:app=>"Podio Bot", :match=>{"regex"=>"^Podio/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podio/1.0"}]}}, {:app=>"Podnews", :match=>{"regex"=>"PodnewsBot"}, :platform=>"bot"}, {:app=>"Podnods Bot", :match=>{"regex"=>"(podnods-crawler|podnods)"}, :platform=>"bot"}, {:app=>"Podscribe", :match=>{"regex"=>"(^Adswizz-podscribe/|^Podscribe/)"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Adswizz-podscribe/1.0"}, {"userAgent"=>"Podscribe/1.1"}]}}, {:app=>"Podverse Feed Parser", :match=>{"regex"=>"^Podverse/Feed Parser"}, :platform=>"bot"}, {:app=>"PodvineBot", :match=>{"regex"=>"^PodvineBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodvineBot/4.0.1 (www.podvine.com)"}]}}, {:app=>"PostRank Bot", :match=>{"regex"=>"^PostRank/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PostRank/2.0 (postrank.com; 1 subscribers)"}]}}, {:app=>"PodwatchPro", :match=>{"regex"=>"Podwatch-Pro Crawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podwatch-Pro Crawler"}]}}, {:app=>"Puppeteer", :match=>{"regex"=>" HeadlessChrome/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/105.0.5173.0 Safari/537.36"}]}}, {:app=>"python-requests", :match=>{"regex"=>"python-requests"}, :platform=>"bot"}, {:app=>"Qiniu spider", :match=>{"regex"=>"qiniu.*spider"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"z0-qiniu-imgstg-spider-1.0"}]}}, {:app=>"RedCircle", :match=>{"regex"=>"RedCircle"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"RedCircle"}]}}, {:app=>"Ridder Bot", :match=>{"regex"=>" RidderBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co)"}]}}, {:app=>"RSS to Telegram", :match=>{"regex"=>"^RSStT"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"RSStT/2.2.1 RSS Reader"}]}}, {:app=>"RSSRadio", :match=>{"regex"=>"^RSSRadio \\("}, :platform=>"bot"}, {:app=>"Ruby Mechanize", :match=>{"regex"=>"^Mechanize|[ -]Mechanize/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Mechanize/2.7.4)"}, {"userAgent"=>"WWW-Mechanize/1.72"}]}}, {:app=>"Screaming Frog SEO Spider", :match=>{"regex"=>"^Screaming Frog "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Screaming Frog SEO Spider/5.1"}]}}, {:app=>"SearchAtlas.com SEO Crawler", :match=>{"regex"=>"^SearchAtlas.*Crawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SearchAtlas.com SEO Crawler"}]}}, {:app=>"SEMrushBot", :match=>{"regex"=>"SemrushBot/|^SEMrushBot$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; SemrushBot/6~bl; http://www.semrush.com/bot.html)"}, {"userAgent"=>"SEMrushBot"}]}}, {:app=>"Serendeputy", :match=>{"regex"=>"SerendeputyBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)"}]}}, {:app=>"Simplecast", :match=>{"regex"=>"^Simplecast$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Simplecast"}]}}, {:app=>"Slack Bot", :match=>{"regex"=>"^Slackbot 1\\.0"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Slackbot 1.0 ( https://api.slack.com/robots)"}]}}, {:app=>"Snapchat Bot", :match=>{"regex"=>"://developers\\.snap\\.com/robots"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots"}]}}, {:app=>"SoundOn Bot", :match=>{"regex"=>"^SoundOn/[\\d.]+\\s+\\(bot"}, :platform=>"bot"}, {:app=>"Spotify cache service", :match=>{"regex"=>"^Spotify/1\\.0$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Spotify/1.0"}]}}, {:app=>"Stitcher Bot", :match=>{"regex"=>"^StitcherBot"}, :platform=>"bot"}, {:app=>"Substack Content Fetcher", :match=>{"regex"=>"^SubstackContentFetch/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SubstackContentFetch/1.0 (https://substack.com/)"}]}}, {:app=>"SupportingCast", :match=>{"regex"=>"^SupportingCast(/.*)?$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SupportingCast/1.0"}, {"userAgent"=>"SupportingCast"}]}}, {:app=>"Timpi search crawler", :match=>{"regex"=>"Timpibot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Timpibot/0.8 ( http://www.timpi.io)"}]}}, {:app=>"Tiny Tiny RSS", :match=>{"regex"=>"^Tiny Tiny RSS/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Tiny Tiny RSS/22.12-c30b24d09 (Unsupported) (https://tt-rss.org/)"}]}}, {:app=>"Trendsmap Resolver", :match=>{"regex"=>"TrendsmapResolver/"}, :platform=>"bot"}, {:app=>"Twingly Bot", :match=>{"regex"=>"Twingly Recon;"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Twingly Recon; twingly.com)"}]}}, {:app=>"Twitterbot", :match=>{"regex"=>"^Twitterbot"}, :platform=>"bot"}, {:app=>"Typhoeus", :match=>{"regex"=>"^Typhoeus"}, :platform=>"bot"}, {:app=>"UCast", :match=>{"regex"=>"^UCast/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"UCast/1.0 Podcast Sync (1 subscribers; feed-id=aHR0cHM6Ly9mZWVkcy5idXp6c3Byb3V0LmNvbS8yMDg0OTQucnNzEAEBAD32-0242-42AC-8583-21E9BAD8C544; +http://www.ucastapp.com/)"}]}}, {:app=>"TelegramBot", :match=>{"regex"=>"^TelegramBot "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"TelegramBot (like TwitterBot)"}]}}, {:app=>"theTradeDesk Content Web Scraper", :match=>{"regex"=>"TTD-Content"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; TTD-Content; +https://www.thetradedesk.com/general/ttd-content)"}]}}, {:app=>"Veritone Engine Toolkit", :match=>{"regex"=>"^veritone/engine-toolkit"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"veritone/engine-toolkit-3.0"}]}}, {:app=>"Vurbl", :match=>{"regex"=>"VurblBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 https://vurbl.com VurblBot/1.0"}]}}, {:app=>"Wget", :match=>{"regex"=>"Wget"}, :platform=>"bot"}, {:app=>"weborama", :match=>{"regex"=>"^weborama-fetcher"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"weborama-fetcher (+http://www.weborama.com)"}]}}, {:app=>"Windows Crawler", :match=>{"regex"=>"^ZDM/.*Windows"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"ZDM/4.0; Windows Mobile 7.0;"}]}}, {:app=>"Windows PowerShell", :match=>{"regex"=>"WindowsPowerShell/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Windows NT; Windows NT 10.0; en-US) WindowsPowerShell/5.1.19041.2673"}]}}, {:app=>"WordPress", :match=>{"regex"=>"^WordPress"}, :platform=>"bot"}, {:app=>"wsrv.nl", :match=>{"regex"=>"https?://wsrv.nl/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; ImageFetcher/9.0; +http://wsrv.nl/)"}]}}, {:app=>"YaCy", :match=>{"regex"=>"^yacybot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"yacybot (/global; amd64 Linux 5.9.8-zen1-1-zen; java 1.8.0_265; Europe/de) http://yacy.net/bot.html"}]}}, {:app=>"Yahoo Crawler", :match=>{"regex"=>"Yahoo! Slurp"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Crawler Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"}]}}, {:app=>"YandexBot", :match=>{"regex"=>"YandexBot/"}, :platform=>"bot"}, {:app=>"Zapier", :match=>{"regex"=>"^Zapier$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Zapier"}]}}, {:app=>"Zencast", :match=>{"regex"=>"^Zencastr/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Zencastr/2.0"}]}}].freeze \ No newline at end of file +BOTS_RULES = [{:app=>"AAABot - unknown bot", :match=>{"regex"=>"AAABot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AAABot"}]}}, {:app=>"Adswizz", :match=>{"regex"=>"^AIS VirtualListener"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AIS VirtualListener"}]}}, {:app=>"AhrefsBot", :match=>{"regex"=>"AhrefsBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; AhrefsBot/7.0; http://ahrefs.com/robot/)"}, {"userAgent"=>"Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)"}]}}, {:app=>"AirableBot", :match=>{"regex"=>"AirableBot-Podcast/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AirableBot-Podcast/1.0 (+https//www.airablenow.com)"}, {"userAgent"=>"AirableBot-Podcast/1.0 ( https//www.airablenow.com)"}]}}, {:app=>"Alexa Flash Briefing cache", :match=>{"regex"=>"^AmazonNewsContentService"}, :platform=>"bot"}, {:app=>"AlignaBot", :match=>{"regex"=>"^Alignabot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Alignabot 1.0 (https://www.alignable.com/)"}]}}, {:app=>"Amazon Music Podcasts Bot", :match=>{"regex"=>"^Amazon Music Podcast"}, :platform=>"bot"}, {:app=>"Anchor Importer", :match=>{"regex"=>"AnchorImport"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AnchorImport/1.0"}]}}, {:app=>"Apache HTTPClient", :match=>{"regex"=>"^Apache\\-HttpClient"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Apache-HttpClient/5.1.4 (Java/1.8.0_144)"}, {"userAgent"=>"Apache-HttpClient/5.1.4 (Java/1.8.0_91)"}]}}, {:app=>"Apple Podcasts (Watch)", :match=>{"regex"=>"^atc/|\\(null\\) watchOS/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"atc/1.0"}, {"userAgent"=>"atc/1.0 watchOS/6.2 model/Watch3,3 hwp/t8004 build/17T529 (6; dt:155)"}, {"userAgent"=>"atc/1.0 watchOS/6.2.8 model/Watch2,3 hwp/t8002 build/17U63 (6; dt:133)"}, {"userAgent"=>"atc/1.0 watchOS/6.2.8 model/Watch3,3 hwp/t8004 build/17U63 (6; dt:155)"}, {"userAgent"=>"atc/1.0 watchOS/6.2.8 model/Watch4,2 hwp/t8006 build/17U63 (6; dt:191)"}, {"userAgent"=>"atc/1.0 watchOS/7.0.2 model/Watch5,10 hwp/t8006 build/18R402 (6; dt:233)"}, {"userAgent"=>"atc/1.0 watchOS/7.0.2 model/Watch5,11 hwp/t8006 build/18R402 (6; dt:234)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch4,2 hwp/t8006 build/18R590 (6; dt:191)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch4,3 hwp/t8006 build/18R590 (6; dt:192)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch4,4 hwp/t8006 build/18R590 (6; dt:193)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch5,1 hwp/t8006 build/18R590 (6; dt:201)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch5,3 hwp/t8006 build/18R590 (6; dt:202)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch5,4 hwp/t8006 build/18R590 (6; dt:202)"}, {"userAgent"=>"(null)/(null) watchOS/5.0.1 model/Watch3,3 hwp/t8004 build/16R381 (6; dt:155)"}]}}, {:app=>"Apple Podcasts automated checks", :match=>{"regex"=>"(iTMS|itunesstored|itms)"}, :platform=>"bot"}, {:app=>"Applebot", :match=>{"regex"=>"^Applebot/"}, :platform=>"bot"}, {:app=>"Archive.org", :match=>{"regex"=>"archive\\.org_bot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; archive.org_bot http://archive.org/details/archive.org_bot)"}]}}, {:app=>"atheerfm", :match=>{"regex"=>"^atheerfm/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"atheerfm/1 CFNetwork/758.3.15 Darwin/15.3.0"}]}}, {:app=>"Audiomack", :match=>{"regex"=>"^Audiomack Podcast Processor/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Audiomack Podcast Processor/1.0 (https://audiomack.com/)"}]}}, {:app=>"AudioWave feed parser", :match=>{"regex"=>"^AudioWaveBot/1\\.0"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AudioWaveBot/1.0"}]}}, {:app=>"AwarioSmartBot", :match=>{"regex"=>"^AwarioSmartBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)"}]}}, {:app=>"Babbar", :match=>{"regex"=>"Barkrowler/"}, :platform=>"bot"}, {:app=>"Baidu", :match=>{"regex"=>"\\(ce\\.baidu\\.com"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Baidu-YunGuanCe-SLABot(ce.baidu.com)"}]}}, {:app=>"bbot", :match=>{"regex"=>"^bbot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"bbot/0.1"}]}}, {:app=>"British Library", :match=>{"regex"=>"^bl\\.uk_ldfc_bot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"bl.uk_ldfc_bot/3.4.0-20200518 ( http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)"}]}}, {:app=>"Blubrry Migration Service", :match=>{"regex"=>"^Blubrry Migration Service"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Blubrry Migration Service"}]}}, {:app=>"Buzzsprout Importer", :match=>{"regex"=>"^Buzzsprout Importer"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Buzzsprout Importer"}]}}, {:app=>"CastFeedValidator", :match=>{"regex"=>"^CastFeedValidator/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"CastFeedValidator/3.0.5 (https://castfeedvalidator.com)"}]}}, {:app=>"Castopod", :match=>{"regex"=>"Castopod/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Castopod/1.0"}, {"userAgent"=>"Castopod/1.1.2"}]}}, {:app=>"Chartable", :match=>{"regex"=>"^Trackable/"}, :platform=>"bot"}, {:app=>"Clark-Crawler, unknown", :match=>{"regex"=>"^clark-crawler2"}, :platform=>"bot"}, {:app=>"Critical Mention", :match=>{"regex"=>"^Podcast-CriticalMention/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcast-CriticalMention/1.0"}]}}, {:app=>"curl", :match=>{"regex"=>"^curl|^libcurl/|^PycURL/| curl/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"libcurl/7.65.1"}, {"userAgent"=>"PycURL/7.43.0 libcurl/7.47.0 GnuTLS/3.4.10 zlib/1.2.8 libidn/1.32 librtmp/2.3"}, {"userAgent"=>"GuzzleHttp/6.1.0 curl/7.50.1 PHP/7.0.13-0ubuntu0.16.10.1"}]}}, {:app=>"DataforSEO", :match=>{"regex"=>"DataForSeoBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)"}]}}, {:app=>"Datagnion Bot", :match=>{"regex"=>"^datagnionbot"}, :platform=>"bot"}, {:app=>"dataprovider.com", :match=>{"regex"=>"Dataprovider\\.com"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Dataprovider.com)"}]}}, {:app=>"Daum", :match=>{"regex"=>"http://cs\\.daum\\.net"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)"}]}}, {:app=>"Deezer Podcasters", :match=>{"regex"=>"^Deezer Podcasters/1\\.0"}, :platform=>"bot"}, {:app=>"Digg", :match=>{"regex"=>"^Digg "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Digg Mozilla/5.0 (Digg/1.0; support@digg.com)"}]}}, {:app=>"dorada", :match=>{"regex"=>"support@dorada\\.co\\.uk"}, :platform=>"bot"}, {:app=>"DotBot", :match=>{"regex"=>"DotBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)"}, {"userAgent"=>"Mozilla/5.0 (compatible; DotBot/1.2; https://opensiteexplorer.org/dotbot; help@moz.com)"}]}}, {:app=>"Downcast Bot", :match=>{"regex"=>"downcast feed consumer/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"downcast feed consumer/0.0.175; (mode=dev; id=u2NgjBSPM6; downcast.fm)"}]}}, {:app=>"DuckDuckBot", :match=>{"regex"=>"DuckDuckBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)"}]}}, {:app=>"F-Secure Riddler", :match=>{"regex"=>"^Riddler "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Riddler (http://riddler.io/about)"}]}}, {:app=>"Facebook Bot", :match=>{"regex"=>"FacebookBot|f?acebookexternalhit/?|^podcastbot$|Facebot|facebookexternalua|^facebookplatform/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"facebookexternalhit/1.1 ( http://www.facebook.com/externalhit_uatext.php)"}, {"userAgent"=>"podcastbot"}, {"userAgent"=>"facebookexternalhit"}, {"userAgent"=>"Facebot"}, {"userAgent"=>"facebookexternalua"}, {"userAgent"=>"facebookplatform/1.0 (+http://developers.facebook.com)"}, {"userAgent"=>"acebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)"}]}}, {:app=>"Feedly", :match=>{"regex"=>"^Feedly/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)"}]}}, {:app=>"Feedspot Fetcher", :match=>{"regex"=>"https://www\\.feedspot\\.com/fs/fetcher"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)"}]}}, {:app=>"FlexGet", :match=>{"regex"=>"^FlexGet/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"FlexGet/3.3.4 (www.flexget.com)"}]}}, {:app=>"Flipboard Proxy", :match=>{"regex"=>"FlipboardProxy/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)"}]}}, {:app=>"Fyyd", :match=>{"regex"=>"^fyyd-poll"}, :platform=>"bot"}, {:app=>"Go Storage Gateway V1", :match=>{"regex"=>"^storagegw-v1-go$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"storagegw-v1-go"}]}}, {:app=>"Goodpods Bot", :match=>{"regex"=>"Goodpods/\\d+\\.\\d+"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Goodpods/2.2"}]}}, {:app=>"Google Adsense Bot", :match=>{"regex"=>"Mediapartners-Google"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mediapartners-Google"}]}}, {:app=>"Google AdsBot", :match=>{"regex"=>"AdsBot-Google"}, :platform=>"bot"}, {:app=>"Google Podcasts Manager", :match=>{"regex"=>"Google-Podcast"}, :platform=>"bot"}, {:app=>"Googlebot", :match=>{"regex"=>"Googlebot/|Googlebot-Video/|Googlebot-Image/|^Feedfetcher-Google|google-xrawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; http://www.google.com/bot.html)"}, {"userAgent"=>"Googlebot-Image/1.0"}, {"userAgent"=>"Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=4653447469100830145)"}, {"userAgent"=>"google-xrawler"}]}}, {:app=>"Gumball", :match=>{"regex"=>"^Gumball"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Gumball.fm Analytics Prefix Checker"}]}}, {:app=>"Headliner", :match=>{"regex"=>"Headliner/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Headliner/1.0.0 +https://headliner.app"}]}}, {:app=>"HubSpot Crawler", :match=>{"regex"=>"HubSpot Crawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; HubSpot Crawler; +https://www.hubspot.com)"}]}}, {:app=>"Internet Archive", :match=>{"regex"=>"Archive-It;|web\\.archive\\.org"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; special_archiver; Archive-It; http://archive-it.org/files/site-owners-special.html)"}, {"userAgent"=>"ia_archiver-web.archive.org"}]}}, {:app=>"Jaunt", :match=>{"regex"=>"^Jaunt/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Jaunt/1.5"}]}}, {:app=>"l'Institut national de l'audiovisuel", :match=>{"regex"=>"INA dlweb"}, :platform=>"bot"}, {:app=>"Libsyn", :match=>{"regex"=>"^Libsyn4"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Libsyn4-Download"}]}}, {:app=>"libwww-perl", :match=>{"regex"=>"^libwww-perl| libwww-perl"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"LWP::Simple/6.34 libwww-perl/6.34"}]}}, {:app=>"Livelap Crawler", :match=>{"regex"=>"LivelapBot"}, :platform=>"bot"}, {:app=>"LTX71", :match=>{"regex"=>"^ltx71 "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"ltx71 - (http://ltx71.com/)"}]}}, {:app=>"MauiBot", :match=>{"regex"=>"^MauiBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"MauiBot (crawler.feedback dc@gmail.com)"}]}}, {:app=>"Mastodon Bot", :match=>{"regex"=>"rb/.*Mastodon/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"http.rb/5.0.4 (Mastodon/3.5.3; +https://mastodon.xyz/) Bot"}]}}, {:app=>"Microsoft Bingbot", :match=>{"regex"=>"(BingPreview/|adidxbot/|[bB]ingbot/)"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534 (KHTML, like Gecko) BingPreview/1.0b"}, {"userAgent"=>"Mozilla/5.0 (compatible; Bingbot/2.0; +http://www.bing.com/bingbot.htm)"}]}}, {:app=>"Microsoft Office", :match=>{"regex"=>"ms-office; MSOffice"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/4.0 (compatible; ms-office; MSOffice 16)"}]}}, {:app=>"Mixcloud Podcast Importer", :match=>{"regex"=>"^MixcloudPodcastImporter/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"MixcloudPodcastImporter/2.0"}]}}, {:app=>"MJ12bot", :match=>{"regex"=>".*MJ12bot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)"}]}}, {:app=>"Mozilla Bot", :match=>{"regex"=>"^'?Mozilla(/5\\.0(\\.\\.\\.)?)?$|^\\(Mozilla/5\\.0\\)$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0"}, {"userAgent"=>"Mozilla"}, {"userAgent"=>"Mozilla/5.0..."}, {"userAgent"=>"'Mozilla/5.0"}, {"userAgent"=>"(Mozilla/5.0)"}]}}, {:app=>"MSN Bot", :match=>{"regex"=>"^msnbot/"}, :platform=>"bot"}, {:app=>"Neevabot", :match=>{"regex"=>".*Neevabot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Neevabot/1.0; https://neeva.com/neevabot)"}]}}, {:app=>"Netcraft Survey Agent", :match=>{"regex"=>" NetcraftSurveyAgent/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)"}]}}, {:app=>"Newspaper", :match=>{"regex"=>"^newspaper/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"newspaper/0.3.0"}]}}, {:app=>"OgScrper", :match=>{"regex"=>"OgScrper"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"OgScrper"}]}}, {:app=>"OkDownload", :match=>{"regex"=>"^OkDownload/"}, :platform=>"bot"}, {:app=>"OP3 Fetcher", :match=>{"regex"=>"^op3-fetcher/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"op3-fetcher/1.0 (https://op3.dev)"}]}}, {:app=>"Overcast feed parser", :match=>{"regex"=>"^Overcast/1\\.0 Podcast Sync"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Overcast/1.0 Podcast Sync"}]}}, {:app=>"Pandora RSS crawler", :match=>{"regex"=>"^PandoraRSSCrawler"}, :platform=>"bot"}, {:app=>"PaperLi", :match=>{"regex"=>"PaperLiBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)"}]}}, {:app=>"PetalBot", :match=>{"regex"=>"PetalBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot; https://aspiegel.com/petalbot)"}]}}, {:app=>"Pingdom", :match=>{"regex"=>"^Pingdom"}, :platform=>"bot"}, {:app=>"PlayerFM Podcast Sync", :match=>{"regex"=>"PlayerFM/.* Podcast Sync"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PlayerFM/1.0 Podcast Sync (0 subscribers; url=https://player.fm/series/series-2567872)"}]}}, {:app=>"Podbean Importer", :match=>{"regex"=>"^Podbean Importer"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podbean Importer"}]}}, {:app=>"Podcastindex.org", :match=>{"regex"=>"^Podcastindex\\.org/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcastindex.org/v0.3.3 (Aggrivate)"}]}}, {:app=>"Podcast de facto Standard", :match=>{"regex"=>"^PodcastStandard/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodcastStandard/0.1.0 (+https://podcast-standard.org)"}]}}, {:app=>"Podcast Archiver", :match=>{"regex"=>"^Podcast%20Archiver/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcast%20Archiver/3000000003 CFNetwork/1406.0.4 Darwin/22.4.0"}]}}, {:app=>"PodCloud", :match=>{"regex"=>"podCloud"}, :platform=>"bot"}, {:app=>"Podcorn", :match=>{"regex"=>"Podcorn/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcorn/1.0"}]}}, {:app=>"PodderBot", :match=>{"regex"=>"PodderBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodderBot/1.0"}]}}, {:app=>"Podfollow", :match=>{"regex"=>"podfollowbot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 https://podfollow.com/crawling podfollowbot/1.0"}]}}, {:app=>"Podgrab", :match=>{"regex"=>"^Podgrab$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podgrab"}]}}, {:app=>"Podhound", :match=>{"regex"=>"PodhoundBeta"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodhoundBeta"}]}}, {:app=>"Podio Bot", :match=>{"regex"=>"^Podio/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podio/1.0"}]}}, {:app=>"Podnews", :match=>{"regex"=>"PodnewsBot"}, :platform=>"bot"}, {:app=>"Podnods Bot", :match=>{"regex"=>"(podnods-crawler|podnods)"}, :platform=>"bot"}, {:app=>"Podscribe", :match=>{"regex"=>"(^Adswizz-podscribe/|^Podscribe/)"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Adswizz-podscribe/1.0"}, {"userAgent"=>"Podscribe/1.1"}]}}, {:app=>"Podverse Feed Parser", :match=>{"regex"=>"^Podverse/Feed Parser"}, :platform=>"bot"}, {:app=>"PodvineBot", :match=>{"regex"=>"^PodvineBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodvineBot/4.0.1 (www.podvine.com)"}]}}, {:app=>"PostRank Bot", :match=>{"regex"=>"^PostRank/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PostRank/2.0 (postrank.com; 1 subscribers)"}]}}, {:app=>"PodwatchPro", :match=>{"regex"=>"Podwatch-Pro Crawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podwatch-Pro Crawler"}]}}, {:app=>"Puppeteer", :match=>{"regex"=>" HeadlessChrome/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/105.0.5173.0 Safari/537.36"}]}}, {:app=>"python-requests", :match=>{"regex"=>"python-requests"}, :platform=>"bot"}, {:app=>"Qiniu spider", :match=>{"regex"=>"qiniu.*spider"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"z0-qiniu-imgstg-spider-1.0"}]}}, {:app=>"RedCircle", :match=>{"regex"=>"RedCircle"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"RedCircle"}]}}, {:app=>"Ridder Bot", :match=>{"regex"=>" RidderBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co)"}]}}, {:app=>"RSS to Telegram", :match=>{"regex"=>"^RSStT"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"RSStT/2.2.1 RSS Reader"}]}}, {:app=>"RSSRadio", :match=>{"regex"=>"^RSSRadio \\("}, :platform=>"bot"}, {:app=>"Ruby Mechanize", :match=>{"regex"=>"^Mechanize|[ -]Mechanize/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Mechanize/2.7.4)"}, {"userAgent"=>"WWW-Mechanize/1.72"}]}}, {:app=>"Screaming Frog SEO Spider", :match=>{"regex"=>"^Screaming Frog "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Screaming Frog SEO Spider/5.1"}]}}, {:app=>"SearchAtlas.com SEO Crawler", :match=>{"regex"=>"^SearchAtlas.*Crawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SearchAtlas.com SEO Crawler"}]}}, {:app=>"SEMrushBot", :match=>{"regex"=>"SemrushBot/|^SEMrushBot$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; SemrushBot/6~bl; http://www.semrush.com/bot.html)"}, {"userAgent"=>"SEMrushBot"}]}}, {:app=>"Serendeputy", :match=>{"regex"=>"SerendeputyBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)"}]}}, {:app=>"Simplecast", :match=>{"regex"=>"^Simplecast$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Simplecast"}]}}, {:app=>"Slack Bot", :match=>{"regex"=>"^Slackbot 1\\.0"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Slackbot 1.0 ( https://api.slack.com/robots)"}]}}, {:app=>"Snapchat Bot", :match=>{"regex"=>"://developers\\.snap\\.com/robots"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots"}]}}, {:app=>"SoundOn Bot", :match=>{"regex"=>"^SoundOn/[\\d.]+\\s+\\(bot"}, :platform=>"bot"}, {:app=>"Spotify cache service", :match=>{"regex"=>"^Spotify/1\\.0$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Spotify/1.0"}]}}, {:app=>"Stitcher Bot", :match=>{"regex"=>"^StitcherBot"}, :platform=>"bot"}, {:app=>"Substack Content Fetcher", :match=>{"regex"=>"^SubstackContentFetch/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SubstackContentFetch/1.0 (https://substack.com/)"}]}}, {:app=>"SupportingCast", :match=>{"regex"=>"^SupportingCast(/.*)?$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SupportingCast/1.0"}, {"userAgent"=>"SupportingCast"}]}}, {:app=>"Timpi search crawler", :match=>{"regex"=>"Timpibot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Timpibot/0.8 ( http://www.timpi.io)"}]}}, {:app=>"Tiny Tiny RSS", :match=>{"regex"=>"^Tiny Tiny RSS/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Tiny Tiny RSS/22.12-c30b24d09 (Unsupported) (https://tt-rss.org/)"}]}}, {:app=>"Trendsmap Resolver", :match=>{"regex"=>"TrendsmapResolver/"}, :platform=>"bot"}, {:app=>"Twingly Bot", :match=>{"regex"=>"Twingly Recon;"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Twingly Recon; twingly.com)"}]}}, {:app=>"Twitterbot", :match=>{"regex"=>"^Twitterbot"}, :platform=>"bot"}, {:app=>"Typhoeus", :match=>{"regex"=>"^Typhoeus"}, :platform=>"bot"}, {:app=>"UCast", :match=>{"regex"=>"^UCast/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"UCast/1.0 Podcast Sync (1 subscribers; feed-id=aHR0cHM6Ly9mZWVkcy5idXp6c3Byb3V0LmNvbS8yMDg0OTQucnNzEAEBAD32-0242-42AC-8583-21E9BAD8C544; +http://www.ucastapp.com/)"}]}}, {:app=>"TelegramBot", :match=>{"regex"=>"^TelegramBot "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"TelegramBot (like TwitterBot)"}]}}, {:app=>"theTradeDesk Content Web Scraper", :match=>{"regex"=>"TTD-Content"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; TTD-Content; +https://www.thetradedesk.com/general/ttd-content)"}]}}, {:app=>"Veritone Engine Toolkit", :match=>{"regex"=>"^veritone/engine-toolkit"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"veritone/engine-toolkit-3.0"}]}}, {:app=>"Vurbl", :match=>{"regex"=>"VurblBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 https://vurbl.com VurblBot/1.0"}]}}, {:app=>"Wget", :match=>{"regex"=>"Wget"}, :platform=>"bot"}, {:app=>"weborama", :match=>{"regex"=>"^weborama-fetcher"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"weborama-fetcher (+http://www.weborama.com)"}]}}, {:app=>"Windows Crawler", :match=>{"regex"=>"^ZDM/.*Windows"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"ZDM/4.0; Windows Mobile 7.0;"}]}}, {:app=>"Windows PowerShell", :match=>{"regex"=>"WindowsPowerShell/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Windows NT; Windows NT 10.0; en-US) WindowsPowerShell/5.1.19041.2673"}]}}, {:app=>"WordPress", :match=>{"regex"=>"^WordPress"}, :platform=>"bot"}, {:app=>"wsrv.nl", :match=>{"regex"=>"https?://wsrv.nl/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; ImageFetcher/9.0; +http://wsrv.nl/)"}]}}, {:app=>"YaCy", :match=>{"regex"=>"^yacybot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"yacybot (/global; amd64 Linux 5.9.8-zen1-1-zen; java 1.8.0_265; Europe/de) http://yacy.net/bot.html"}]}}, {:app=>"Yahoo Crawler", :match=>{"regex"=>"Yahoo! Slurp"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Crawler Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"}]}}, {:app=>"YandexBot", :match=>{"regex"=>"YandexBot/"}, :platform=>"bot"}, {:app=>"Zapier", :match=>{"regex"=>"^Zapier$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Zapier"}]}}, {:app=>"Zencast", :match=>{"regex"=>"^Zencastr/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Zencastr/2.0"}]}}].freeze \ No newline at end of file From 750290e6f2b2a8f523eafd1d7474eef9dd3adcb6 Mon Sep 17 00:00:00 2001 From: Robert Wachs Date: Wed, 3 Jul 2024 11:22:57 +0200 Subject: [PATCH 2/2] v1.1.4 --- CHANGELOG.md | 4 ++++ Gemfile.lock | 2 +- lib/pod_ident/version.rb | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a29af40..d059fe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [1.1.4] + +- Detect `Apache-HttpClient/5.1.4 (Java/1.8.0_144)` as bots + ## [1.1.3] - Improve radio.net detection for android devices diff --git a/Gemfile.lock b/Gemfile.lock index 21cb02a..834d572 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - pod_ident (1.1.3) + pod_ident (1.1.4) GEM remote: https://rubygems.org/ diff --git a/lib/pod_ident/version.rb b/lib/pod_ident/version.rb index 1593247..0dfdf59 100644 --- a/lib/pod_ident/version.rb +++ b/lib/pod_ident/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module PodIdent - VERSION = '1.1.3' + VERSION = '1.1.4' end