From d6e14a464a4a2c9cd42983e56e5f985112020f09 Mon Sep 17 00:00:00 2001 From: Robert Wachs Date: Tue, 15 Oct 2024 09:43:20 +0200 Subject: [PATCH] update bot lists, version 1.1.6 --- CHANGELOG.md | 4 + Gemfile.lock | 2 +- Rakefile | 2 +- lib/detection_rules_bots.yml | 195 ++++++++++++++++++- lib/detection_rules_custom_bots.yml | 8 + lib/pod_ident/detection_rules_bots.rb | 2 +- lib/pod_ident/detection_rules_custom_bots.rb | 2 +- lib/pod_ident/version.rb | 2 +- spec/detection_rules_bots.rb | 2 +- spec/detection_rules_custom_bots.rb | 2 +- 10 files changed, 208 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9fb188..58d47e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [1.1.6] + +- Update bot detection list, purpose: detect `agma/*` as bot + ## [1.1.5] - Rename `Amazon Alexa` to `Alexa` diff --git a/Gemfile.lock b/Gemfile.lock index 834d572..e211bf9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - pod_ident (1.1.4) + pod_ident (1.1.6) GEM remote: https://rubygems.org/ diff --git a/Rakefile b/Rakefile index 0d8a851..8d46738 100644 --- a/Rakefile +++ b/Rakefile @@ -27,7 +27,7 @@ task :update_bot_rules do def json_content json = File.open(BOT_JSON).read - content = JSON.parse(json) + JSON.parse(json) end def generate_entry_yml(entry) diff --git a/lib/detection_rules_bots.yml b/lib/detection_rules_bots.yml index 83d08a9..37da303 100644 --- a/lib/detection_rules_bots.yml +++ b/lib/detection_rules_bots.yml @@ -13,6 +13,13 @@ test: userAgents: - userAgent: AIS VirtualListener +- app: ma Podcast (agma) Bot + match: + regex: "^agma/" + platform: bot + test: + userAgents: + - userAgent: agma/1.0 - app: AhrefsBot match: regex: AhrefsBot/ @@ -40,6 +47,13 @@ test: userAgents: - userAgent: Alignabot 1.0 (https://www.alignable.com/) +- app: All Ears + match: + regex: "^AllEars/" + platform: bot + test: + userAgents: + - userAgent: AllEars/1.0; (+www.allears.ai) - app: Amazon Music Podcasts Bot match: regex: "^Amazon Music Podcast" @@ -51,14 +65,14 @@ test: userAgents: - userAgent: AnchorImport/1.0 -- app: Apache HTTPClient +- app: Apple Notes share extension match: - regex: "^Apache\\-HttpClient" + regex: "^com.apple.(mobilenotes.SharingExtension|Notes.SharingExtension)/" platform: bot test: userAgents: - - userAgent: Apache-HttpClient/5.1.4 (Java/1.8.0_144) - - userAgent: Apache-HttpClient/5.1.4 (Java/1.8.0_91) + - userAgent: com.apple.mobilenotes.SharingExtension/2511 CFNetwork/1494.0.7 Darwin/23.4.0 + - userAgent: com.apple.Notes.SharingExtension/2812 CFNetwork/1494.0.7 Darwin/23.4.0 - app: Apple Podcasts (Watch) match: regex: "^atc/|\\(null\\) watchOS/" @@ -143,6 +157,13 @@ test: userAgents: - userAgent: bbot/0.1 +- app: Beacon Podcast Sync + match: + regex: "^Beacon/\\d" + platform: bot + test: + userAgents: + - userAgent: Beacon/1.0 Podcast Sync +https://getbeacons.com/ - app: British Library match: regex: "^bl\\.uk_ldfc_bot" @@ -157,6 +178,13 @@ test: userAgents: - userAgent: Blubrry Migration Service +- app: Bullhorn Crawler + match: + regex: "^BullhornCrawler/" + platform: bot + test: + userAgents: + - userAgent: BullhornCrawler/unknown-version(+http://bullhorn.fm/) - app: Buzzsprout Importer match: regex: "^Buzzsprout Importer" @@ -164,6 +192,14 @@ test: userAgents: - userAgent: Buzzsprout Importer +- app: Bytespider.*bytedance + match: + regex: Bytespider + platform: bot + test: + userAgents: + - userAgent: Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like + Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com) - app: CastFeedValidator match: regex: "^CastFeedValidator/" @@ -187,6 +223,14 @@ match: regex: "^clark-crawler2" platform: bot +- app: Claudebot + match: + regex: ClaudeBot/ + platform: bot + test: + userAgents: + - userAgent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; + +claudebot@anthropic.com) - app: Critical Mention match: regex: "^Podcast-CriticalMention/" @@ -229,6 +273,13 @@ test: userAgents: - userAgent: Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966) +- app: Deepcast Podcast Sync + match: + regex: "^DeepCast/\\d.*Sync" + platform: bot + test: + userAgents: + - userAgent: DeepCast/4.3.0 Podcast Sync (+https://deepcast.fm) - app: Deezer Podcasters match: regex: "^Deezer Podcasters/1\\.0" @@ -352,7 +403,7 @@ platform: bot - app: Googlebot match: - regex: Googlebot/|Googlebot-Video/|Googlebot-Image/|^Feedfetcher-Google|google-xrawler + regex: Googlebot/|Googlebot-Video/|Googlebot-Image/|^Feedfetcher-Google|google-xrawler|^Googlebot-News|^Google-Safety$ platform: bot test: userAgents: @@ -363,6 +414,8 @@ - userAgent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=4653447469100830145) - userAgent: google-xrawler + - userAgent: Googlebot-News Audio + - userAgent: Google-Safety - app: Gumball match: regex: "^Gumball" @@ -372,7 +425,7 @@ - userAgent: Gumball.fm Analytics Prefix Checker - app: Headliner match: - regex: Headliner/ + regex: "^Headliner/.*\\+https://headliner\\.app$" platform: bot test: userAgents: @@ -392,6 +445,22 @@ userAgents: - userAgent: Mozilla/5.0 (compatible; special_archiver; Archive-It; http://archive-it.org/files/site-owners-special.html) - userAgent: ia_archiver-web.archive.org +- app: iOS share menu + match: + regex: "^SharingUIService/" + platform: bot + test: + userAgents: + - userAgent: SharingUIService/1 CFNetwork/1467.1 Darwin/23.0.0 + - userAgent: SharingUIService/1 CFNetwork/1490.0.4 Darwin/23.2.0 +- app: IPFS Podcasting Node + match: + regex: "^IPFS Podcasting Node/" + platform: bot + test: + userAgents: + - userAgent: IPFS Podcasting Node/0.6n (+https://ipfspodcasting.net) + - userAgent: IPFS Podcasting Node/0.7p,u,s (+https://ipfspodcasting.net) - app: Jaunt match: regex: "^Jaunt/" @@ -403,6 +472,13 @@ match: regex: INA dlweb platform: bot +- app: Library and Archives Canada + match: + regex: LAC_IAHarvester/ + platform: bot + test: + userAgents: + - userAgent: Mozilla/5.0 (compatible; LAC_IAHarvester/3.3.0; +https://library-archives.canada.ca/eng/services/government-canada/web-social-media-preservation-program/Pages/web-archive.aspx) - app: Libsyn match: regex: "^Libsyn4" @@ -421,6 +497,13 @@ match: regex: LivelapBot platform: bot +- app: ListenNotes + match: + regex: "^ListenNotes/3\\.0 \\(" + platform: bot + test: + userAgents: + - userAgent: ListenNotes/3.0 (id=c1ef144e30174bf6ba1fedf9174ed908; +https://www.listennotes.com/about/) - app: LTX71 match: regex: "^ltx71 " @@ -428,6 +511,14 @@ test: userAgents: - userAgent: ltx71 - (http://ltx71.com/) +- app: MacOS share menu + match: + regex: "^ShareSheetUI/" + platform: bot + test: + userAgents: + - userAgent: ShareSheetUI/1 CFNetwork/1399.4 Darwin/22.1.0 + - userAgent: ShareSheetUI/1 CFNetwork/1474 Darwin/23.0.0 - app: MauiBot match: regex: "^MauiBot" @@ -442,6 +533,13 @@ test: userAgents: - userAgent: http.rb/5.0.4 (Mastodon/3.5.3; +https://mastodon.xyz/) Bot +- app: Metacast Crawler + match: + regex: "^Metacast/.* Crawler" + platform: bot + test: + userAgents: + - userAgent: Metacast/1.0 Crawler +https://metacast.app - app: Microsoft Bingbot match: regex: "(BingPreview/|adidxbot/|[bB]ingbot/)" @@ -526,6 +624,16 @@ test: userAgents: - userAgent: op3-fetcher/1.0 (https://op3.dev) +- app: OpenAI GPTBot + match: + regex: GPTBot/ + platform: bot + test: + userAgents: + - userAgent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; + +https://openai.com/gptbot) + - userAgent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; + +https://openai.com/gptbot) - app: Overcast feed parser match: regex: "^Overcast/1\\.0 Podcast Sync" @@ -533,6 +641,8 @@ test: userAgents: - userAgent: Overcast/1.0 Podcast Sync + - userAgent: Overcast/1.0 Podcast Sync (+http://overcast.fm/) + - userAgent: Overcast/1.0 Podcast Sync (1 subscribers; feed-id=1234567; +http://overcast.fm/) - app: Pandora RSS crawler match: regex: "^PandoraRSSCrawler" @@ -645,6 +755,28 @@ match: regex: "(podnods-crawler|podnods)" platform: bot +- app: Podplay Podcast Sync + match: + regex: "^Podplay-Podcast-Sync/" + platform: bot + test: + userAgents: + - userAgent: Podplay-Podcast-Sync/1.0 (+https://podplay.com/articles/contact) +- app: Podroll Analyzer + match: + regex: "^pdrl\\.fm " + platform: bot + test: + userAgents: + - userAgent: pdrl.fm Analyzer / 1.0.0 +- app: Podscan Bot + match: + regex: "^PodscanBot/" + platform: bot + test: + userAgents: + - userAgent: PodscanBot/1.1.0 Ubuntu/22.04 (Linux) +https://podscan.fm + - userAgent: PodscanBot/1.1.0 Ubuntu/22.04 (Linux) Podscan.fm - app: Podscribe match: regex: "(^Adswizz-podscribe/|^Podscribe/)" @@ -653,6 +785,15 @@ userAgents: - userAgent: Adswizz-podscribe/1.0 - userAgent: Podscribe/1.1 +- app: PodUptime + match: + regex: "(PodUptime|PodUptimeBot)/\\d" + platform: bot + test: + userAgents: + - userAgent: Mozilla/5.0 (compatible; PodUptime/1.0; +https://poduptime.com) + - userAgent: Mozilla/5.0 (compatible; PodUptimeBot/1.0; +https://poduptime.com; + rid:5c02fc9b-2660-4d3e-a352-de1a1aabdc0a) - app: Podverse Feed Parser match: regex: "^Podverse/Feed Parser" @@ -697,6 +838,13 @@ test: userAgents: - userAgent: z0-qiniu-imgstg-spider-1.0 +- app: Radioline Stream Monitoring + match: + regex: "^Radioline Stream Monitoring" + platform: bot + test: + userAgents: + - userAgent: Radioline Stream Monitoring (tech@radioline.co) - app: RedCircle match: regex: RedCircle @@ -704,6 +852,13 @@ test: userAgents: - userAgent: RedCircle +- app: Rephonic + match: + regex: "^Rephonic/\\d" + platform: bot + test: + userAgents: + - userAgent: Rephonic/1.0 - app: Ridder Bot match: regex: " RidderBot/" @@ -784,6 +939,13 @@ match: regex: "^SoundOn/[\\d.]+\\s+\\(bot" platform: bot +- app: Speechmatics ASR + match: + regex: "^Speechmatics-API/" + platform: bot + test: + userAgents: + - userAgent: Speechmatics-API/2.0 - app: Spotify cache service match: regex: "^Spotify/1\\.0$" @@ -824,10 +986,24 @@ test: userAgents: - userAgent: Tiny Tiny RSS/22.12-c30b24d09 (Unsupported) (https://tt-rss.org/) +- app: Transistor + match: + regex: "^Transistor\\.fm/" + platform: bot + test: + userAgents: + - userAgent: Transistor.fm/1.0 - app: Trendsmap Resolver match: regex: TrendsmapResolver/ platform: bot +- app: Triton Digital Podcast Sync + match: + regex: "^Triton Digital Podcast Sync$" + platform: bot + test: + userAgents: + - userAgent: Triton Digital Podcast Sync - app: Twingly Bot match: regex: Twingly Recon; @@ -851,6 +1027,13 @@ userAgents: - userAgent: UCast/1.0 Podcast Sync (1 subscribers; feed-id=aHR0cHM6Ly9mZWVkcy5idXp6c3Byb3V0LmNvbS8yMDg0OTQucnNzEAEBAD32-0242-42AC-8583-21E9BAD8C544; +http://www.ucastapp.com/) +- app: UptimeRobot + match: + regex: UptimeRobot/\d + platform: bot + test: + userAgents: + - userAgent: Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/) - app: TelegramBot match: regex: "^TelegramBot " diff --git a/lib/detection_rules_custom_bots.yml b/lib/detection_rules_custom_bots.yml index 43af747..de93493 100644 --- a/lib/detection_rules_custom_bots.yml +++ b/lib/detection_rules_custom_bots.yml @@ -1,4 +1,12 @@ --- +- app: Apache HTTPClient + match: + regex: "^Apache\\-HttpClient" + platform: bot + test: + userAgents: + - userAgent: Apache-HttpClient/5.1.4 (Java/1.8.0_144) + - userAgent: Apache-HttpClient/5.1.4 (Java/1.8.0_91) - app: curl match: regex: "^Faraday" diff --git a/lib/pod_ident/detection_rules_bots.rb b/lib/pod_ident/detection_rules_bots.rb index 88d67cc..beda38a 100644 --- a/lib/pod_ident/detection_rules_bots.rb +++ b/lib/pod_ident/detection_rules_bots.rb @@ -1,3 +1,3 @@ # DO NOT EDIT THIS FILE - it gets automatically generated by running "bin/parse-rules" -BOTS_RULES = [{:app=>"AAABot - unknown bot", :match=>{"regex"=>"AAABot"}, :platform=>"bot"}, {:app=>"Adswizz", :match=>{"regex"=>"^AIS VirtualListener"}, :platform=>"bot"}, {:app=>"AhrefsBot", :match=>{"regex"=>"AhrefsBot/"}, :platform=>"bot"}, {:app=>"AirableBot", :match=>{"regex"=>"AirableBot-Podcast/"}, :platform=>"bot"}, {:app=>"Alexa Flash Briefing cache", :match=>{"regex"=>"^AmazonNewsContentService"}, :platform=>"bot"}, {:app=>"AlignaBot", :match=>{"regex"=>"^Alignabot"}, :platform=>"bot"}, {:app=>"Amazon Music Podcasts Bot", :match=>{"regex"=>"^Amazon Music Podcast"}, :platform=>"bot"}, {:app=>"Anchor Importer", :match=>{"regex"=>"AnchorImport"}, :platform=>"bot"}, {:app=>"Apache HTTPClient", :match=>{"regex"=>"^Apache\\-HttpClient"}, :platform=>"bot"}, {:app=>"Apple Podcasts (Watch)", :match=>{"regex"=>"^atc/|\\(null\\) watchOS/"}, :platform=>"bot"}, {:app=>"Apple Podcasts automated checks", :match=>{"regex"=>"(iTMS|itunesstored|itms)"}, :platform=>"bot"}, {:app=>"Applebot", :match=>{"regex"=>"^Applebot/"}, :platform=>"bot"}, {:app=>"Archive.org", :match=>{"regex"=>"archive\\.org_bot"}, :platform=>"bot"}, {:app=>"atheerfm", :match=>{"regex"=>"^atheerfm/"}, :platform=>"bot"}, {:app=>"Audiomack", :match=>{"regex"=>"^Audiomack Podcast Processor/"}, :platform=>"bot"}, {:app=>"AudioWave feed parser", :match=>{"regex"=>"^AudioWaveBot/1\\.0"}, :platform=>"bot"}, {:app=>"AwarioSmartBot", :match=>{"regex"=>"^AwarioSmartBot/"}, :platform=>"bot"}, {:app=>"Babbar", :match=>{"regex"=>"Barkrowler/"}, :platform=>"bot"}, {:app=>"Baidu", :match=>{"regex"=>"\\(ce\\.baidu\\.com"}, :platform=>"bot"}, {:app=>"bbot", :match=>{"regex"=>"^bbot/"}, :platform=>"bot"}, {:app=>"British Library", :match=>{"regex"=>"^bl\\.uk_ldfc_bot"}, :platform=>"bot"}, {:app=>"Blubrry Migration Service", :match=>{"regex"=>"^Blubrry Migration Service"}, :platform=>"bot"}, {:app=>"Buzzsprout Importer", :match=>{"regex"=>"^Buzzsprout Importer"}, :platform=>"bot"}, {:app=>"CastFeedValidator", :match=>{"regex"=>"^CastFeedValidator/"}, :platform=>"bot"}, {:app=>"Castopod", :match=>{"regex"=>"Castopod/\\d"}, :platform=>"bot"}, {:app=>"Chartable", :match=>{"regex"=>"^Trackable/"}, :platform=>"bot"}, {:app=>"Clark-Crawler, unknown", :match=>{"regex"=>"^clark-crawler2"}, :platform=>"bot"}, {:app=>"Critical Mention", :match=>{"regex"=>"^Podcast-CriticalMention/"}, :platform=>"bot"}, {:app=>"curl", :match=>{"regex"=>"^curl|^libcurl/|^PycURL/| curl/"}, :platform=>"bot"}, {:app=>"DataforSEO", :match=>{"regex"=>"DataForSeoBot/"}, :platform=>"bot"}, {:app=>"Datagnion Bot", :match=>{"regex"=>"^datagnionbot"}, :platform=>"bot"}, {:app=>"dataprovider.com", :match=>{"regex"=>"Dataprovider\\.com"}, :platform=>"bot"}, {:app=>"Daum", :match=>{"regex"=>"http://cs\\.daum\\.net"}, :platform=>"bot"}, {:app=>"Deezer Podcasters", :match=>{"regex"=>"^Deezer Podcasters/1\\.0"}, :platform=>"bot"}, {:app=>"Digg", :match=>{"regex"=>"^Digg "}, :platform=>"bot"}, {:app=>"dorada", :match=>{"regex"=>"support@dorada\\.co\\.uk"}, :platform=>"bot"}, {:app=>"DotBot", :match=>{"regex"=>"DotBot"}, :platform=>"bot"}, {:app=>"Downcast Bot", :match=>{"regex"=>"downcast feed consumer/"}, :platform=>"bot"}, {:app=>"DuckDuckBot", :match=>{"regex"=>"DuckDuckBot"}, :platform=>"bot"}, {:app=>"F-Secure Riddler", :match=>{"regex"=>"^Riddler "}, :platform=>"bot"}, {:app=>"Facebook Bot", :match=>{"regex"=>"FacebookBot|f?acebookexternalhit/?|^podcastbot$|Facebot|facebookexternalua|^facebookplatform/"}, :platform=>"bot"}, {:app=>"Feedly", :match=>{"regex"=>"^Feedly/"}, :platform=>"bot"}, {:app=>"Feedspot Fetcher", :match=>{"regex"=>"https://www\\.feedspot\\.com/fs/fetcher"}, :platform=>"bot"}, {:app=>"FlexGet", :match=>{"regex"=>"^FlexGet/"}, :platform=>"bot"}, {:app=>"Flipboard Proxy", :match=>{"regex"=>"FlipboardProxy/"}, :platform=>"bot"}, {:app=>"Fyyd", :match=>{"regex"=>"^fyyd-poll"}, :platform=>"bot"}, {:app=>"Go Storage Gateway V1", :match=>{"regex"=>"^storagegw-v1-go$"}, :platform=>"bot"}, {:app=>"Goodpods Bot", :match=>{"regex"=>"Goodpods/\\d+\\.\\d+"}, :platform=>"bot"}, {:app=>"Google Adsense Bot", :match=>{"regex"=>"Mediapartners-Google"}, :platform=>"bot"}, {:app=>"Google AdsBot", :match=>{"regex"=>"AdsBot-Google"}, :platform=>"bot"}, {:app=>"Google Podcasts Manager", :match=>{"regex"=>"Google-Podcast"}, :platform=>"bot"}, {:app=>"Googlebot", :match=>{"regex"=>"Googlebot/|Googlebot-Video/|Googlebot-Image/|^Feedfetcher-Google|google-xrawler"}, :platform=>"bot"}, {:app=>"Gumball", :match=>{"regex"=>"^Gumball"}, :platform=>"bot"}, {:app=>"Headliner", :match=>{"regex"=>"Headliner/"}, :platform=>"bot"}, {:app=>"HubSpot Crawler", :match=>{"regex"=>"HubSpot Crawler"}, :platform=>"bot"}, {:app=>"Internet Archive", :match=>{"regex"=>"Archive-It;|web\\.archive\\.org"}, :platform=>"bot"}, {:app=>"Jaunt", :match=>{"regex"=>"^Jaunt/"}, :platform=>"bot"}, {:app=>"l'Institut national de l'audiovisuel", :match=>{"regex"=>"INA dlweb"}, :platform=>"bot"}, {:app=>"Libsyn", :match=>{"regex"=>"^Libsyn4"}, :platform=>"bot"}, {:app=>"libwww-perl", :match=>{"regex"=>"^libwww-perl| libwww-perl"}, :platform=>"bot"}, {:app=>"Livelap Crawler", :match=>{"regex"=>"LivelapBot"}, :platform=>"bot"}, {:app=>"LTX71", :match=>{"regex"=>"^ltx71 "}, :platform=>"bot"}, {:app=>"MauiBot", :match=>{"regex"=>"^MauiBot"}, :platform=>"bot"}, {:app=>"Mastodon Bot", :match=>{"regex"=>"rb/.*Mastodon/"}, :platform=>"bot"}, {:app=>"Microsoft Bingbot", :match=>{"regex"=>"(BingPreview/|adidxbot/|[bB]ingbot/)"}, :platform=>"bot"}, {:app=>"Microsoft Office", :match=>{"regex"=>"ms-office; MSOffice"}, :platform=>"bot"}, {:app=>"Mixcloud Podcast Importer", :match=>{"regex"=>"^MixcloudPodcastImporter/"}, :platform=>"bot"}, {:app=>"MJ12bot", :match=>{"regex"=>".*MJ12bot"}, :platform=>"bot"}, {:app=>"Mozilla Bot", :match=>{"regex"=>"^'?Mozilla(/5\\.0(\\.\\.\\.)?)?$|^\\(Mozilla/5\\.0\\)$"}, :platform=>"bot"}, {:app=>"MSN Bot", :match=>{"regex"=>"^msnbot/"}, :platform=>"bot"}, {:app=>"Neevabot", :match=>{"regex"=>".*Neevabot"}, :platform=>"bot"}, {:app=>"Netcraft Survey Agent", :match=>{"regex"=>" NetcraftSurveyAgent/"}, :platform=>"bot"}, {:app=>"Newspaper", :match=>{"regex"=>"^newspaper/\\d"}, :platform=>"bot"}, {:app=>"OgScrper", :match=>{"regex"=>"OgScrper"}, :platform=>"bot"}, {:app=>"OkDownload", :match=>{"regex"=>"^OkDownload/"}, :platform=>"bot"}, {:app=>"OP3 Fetcher", :match=>{"regex"=>"^op3-fetcher/"}, :platform=>"bot"}, {:app=>"Overcast feed parser", :match=>{"regex"=>"^Overcast/1\\.0 Podcast Sync"}, :platform=>"bot"}, {:app=>"Pandora RSS crawler", :match=>{"regex"=>"^PandoraRSSCrawler"}, :platform=>"bot"}, {:app=>"PaperLi", :match=>{"regex"=>"PaperLiBot/"}, :platform=>"bot"}, {:app=>"PetalBot", :match=>{"regex"=>"PetalBot"}, :platform=>"bot"}, {:app=>"Pingdom", :match=>{"regex"=>"^Pingdom"}, :platform=>"bot"}, {:app=>"PlayerFM Podcast Sync", :match=>{"regex"=>"PlayerFM/.* Podcast Sync"}, :platform=>"bot"}, {:app=>"Podbean Importer", :match=>{"regex"=>"^Podbean Importer"}, :platform=>"bot"}, {:app=>"Podcastindex.org", :match=>{"regex"=>"^Podcastindex\\.org/"}, :platform=>"bot"}, {:app=>"Podcast de facto Standard", :match=>{"regex"=>"^PodcastStandard/"}, :platform=>"bot"}, {:app=>"Podcast Archiver", :match=>{"regex"=>"^Podcast%20Archiver/"}, :platform=>"bot"}, {:app=>"PodCloud", :match=>{"regex"=>"podCloud"}, :platform=>"bot"}, {:app=>"Podcorn", :match=>{"regex"=>"Podcorn/"}, :platform=>"bot"}, {:app=>"PodderBot", :match=>{"regex"=>"PodderBot/"}, :platform=>"bot"}, {:app=>"Podfollow", :match=>{"regex"=>"podfollowbot/"}, :platform=>"bot"}, {:app=>"Podgrab", :match=>{"regex"=>"^Podgrab$"}, :platform=>"bot"}, {:app=>"Podhound", :match=>{"regex"=>"PodhoundBeta"}, :platform=>"bot"}, {:app=>"Podio Bot", :match=>{"regex"=>"^Podio/"}, :platform=>"bot"}, {:app=>"Podnews", :match=>{"regex"=>"PodnewsBot"}, :platform=>"bot"}, {:app=>"Podnods Bot", :match=>{"regex"=>"(podnods-crawler|podnods)"}, :platform=>"bot"}, {:app=>"Podscribe", :match=>{"regex"=>"(^Adswizz-podscribe/|^Podscribe/)"}, :platform=>"bot"}, {:app=>"Podverse Feed Parser", :match=>{"regex"=>"^Podverse/Feed Parser"}, :platform=>"bot"}, {:app=>"PodvineBot", :match=>{"regex"=>"^PodvineBot/"}, :platform=>"bot"}, {:app=>"PostRank Bot", :match=>{"regex"=>"^PostRank/"}, :platform=>"bot"}, {:app=>"PodwatchPro", :match=>{"regex"=>"Podwatch-Pro Crawler"}, :platform=>"bot"}, {:app=>"Puppeteer", :match=>{"regex"=>" HeadlessChrome/\\d"}, :platform=>"bot"}, {:app=>"python-requests", :match=>{"regex"=>"python-requests"}, :platform=>"bot"}, {:app=>"Qiniu spider", :match=>{"regex"=>"qiniu.*spider"}, :platform=>"bot"}, {:app=>"RedCircle", :match=>{"regex"=>"RedCircle"}, :platform=>"bot"}, {:app=>"Ridder Bot", :match=>{"regex"=>" RidderBot/"}, :platform=>"bot"}, {:app=>"RSS to Telegram", :match=>{"regex"=>"^RSStT"}, :platform=>"bot"}, {:app=>"RSSRadio", :match=>{"regex"=>"^RSSRadio \\("}, :platform=>"bot"}, {:app=>"Ruby Mechanize", :match=>{"regex"=>"^Mechanize|[ -]Mechanize/"}, :platform=>"bot"}, {:app=>"Screaming Frog SEO Spider", :match=>{"regex"=>"^Screaming Frog "}, :platform=>"bot"}, {:app=>"SearchAtlas.com SEO Crawler", :match=>{"regex"=>"^SearchAtlas.*Crawler"}, :platform=>"bot"}, {:app=>"SEMrushBot", :match=>{"regex"=>"SemrushBot/|^SEMrushBot$"}, :platform=>"bot"}, {:app=>"Serendeputy", :match=>{"regex"=>"SerendeputyBot/"}, :platform=>"bot"}, {:app=>"Simplecast", :match=>{"regex"=>"^Simplecast$"}, :platform=>"bot"}, {:app=>"Slack Bot", :match=>{"regex"=>"^Slackbot 1\\.0"}, :platform=>"bot"}, {:app=>"Snapchat Bot", :match=>{"regex"=>"://developers\\.snap\\.com/robots"}, :platform=>"bot"}, {:app=>"SoundOn Bot", :match=>{"regex"=>"^SoundOn/[\\d.]+\\s+\\(bot"}, :platform=>"bot"}, {:app=>"Spotify cache service", :match=>{"regex"=>"^Spotify/1\\.0$"}, :platform=>"bot"}, {:app=>"Stitcher Bot", :match=>{"regex"=>"^StitcherBot"}, :platform=>"bot"}, {:app=>"Substack Content Fetcher", :match=>{"regex"=>"^SubstackContentFetch/"}, :platform=>"bot"}, {:app=>"SupportingCast", :match=>{"regex"=>"^SupportingCast(/.*)?$"}, :platform=>"bot"}, {:app=>"Timpi search crawler", :match=>{"regex"=>"Timpibot/"}, :platform=>"bot"}, {:app=>"Tiny Tiny RSS", :match=>{"regex"=>"^Tiny Tiny RSS/"}, :platform=>"bot"}, {:app=>"Trendsmap Resolver", :match=>{"regex"=>"TrendsmapResolver/"}, :platform=>"bot"}, {:app=>"Twingly Bot", :match=>{"regex"=>"Twingly Recon;"}, :platform=>"bot"}, {:app=>"Twitterbot", :match=>{"regex"=>"^Twitterbot"}, :platform=>"bot"}, {:app=>"Typhoeus", :match=>{"regex"=>"^Typhoeus"}, :platform=>"bot"}, {:app=>"UCast", :match=>{"regex"=>"^UCast/"}, :platform=>"bot"}, {:app=>"TelegramBot", :match=>{"regex"=>"^TelegramBot "}, :platform=>"bot"}, {:app=>"theTradeDesk Content Web Scraper", :match=>{"regex"=>"TTD-Content"}, :platform=>"bot"}, {:app=>"Veritone Engine Toolkit", :match=>{"regex"=>"^veritone/engine-toolkit"}, :platform=>"bot"}, {:app=>"Vurbl", :match=>{"regex"=>"VurblBot"}, :platform=>"bot"}, {:app=>"Wget", :match=>{"regex"=>"Wget"}, :platform=>"bot"}, {:app=>"weborama", :match=>{"regex"=>"^weborama-fetcher"}, :platform=>"bot"}, {:app=>"Windows Crawler", :match=>{"regex"=>"^ZDM/.*Windows"}, :platform=>"bot"}, {:app=>"Windows PowerShell", :match=>{"regex"=>"WindowsPowerShell/"}, :platform=>"bot"}, {:app=>"WordPress", :match=>{"regex"=>"^WordPress"}, :platform=>"bot"}, {:app=>"wsrv.nl", :match=>{"regex"=>"https?://wsrv.nl/"}, :platform=>"bot"}, {:app=>"YaCy", :match=>{"regex"=>"^yacybot"}, :platform=>"bot"}, {:app=>"Yahoo Crawler", :match=>{"regex"=>"Yahoo! Slurp"}, :platform=>"bot"}, {:app=>"YandexBot", :match=>{"regex"=>"YandexBot/"}, :platform=>"bot"}, {:app=>"Zapier", :match=>{"regex"=>"^Zapier$"}, :platform=>"bot"}, {:app=>"Zencast", :match=>{"regex"=>"^Zencastr/"}, :platform=>"bot"}].freeze \ No newline at end of file +BOTS_RULES = [{:app=>"AAABot - unknown bot", :match=>{"regex"=>"AAABot"}, :platform=>"bot"}, {:app=>"Adswizz", :match=>{"regex"=>"^AIS VirtualListener"}, :platform=>"bot"}, {:app=>"ma Podcast (agma) Bot", :match=>{"regex"=>"^agma/"}, :platform=>"bot"}, {:app=>"AhrefsBot", :match=>{"regex"=>"AhrefsBot/"}, :platform=>"bot"}, {:app=>"AirableBot", :match=>{"regex"=>"AirableBot-Podcast/"}, :platform=>"bot"}, {:app=>"Alexa Flash Briefing cache", :match=>{"regex"=>"^AmazonNewsContentService"}, :platform=>"bot"}, {:app=>"AlignaBot", :match=>{"regex"=>"^Alignabot"}, :platform=>"bot"}, {:app=>"All Ears", :match=>{"regex"=>"^AllEars/"}, :platform=>"bot"}, {:app=>"Amazon Music Podcasts Bot", :match=>{"regex"=>"^Amazon Music Podcast"}, :platform=>"bot"}, {:app=>"Anchor Importer", :match=>{"regex"=>"AnchorImport"}, :platform=>"bot"}, {:app=>"Apple Notes share extension", :match=>{"regex"=>"^com.apple.(mobilenotes.SharingExtension|Notes.SharingExtension)/"}, :platform=>"bot"}, {:app=>"Apple Podcasts (Watch)", :match=>{"regex"=>"^atc/|\\(null\\) watchOS/"}, :platform=>"bot"}, {:app=>"Apple Podcasts automated checks", :match=>{"regex"=>"(iTMS|itunesstored|itms)"}, :platform=>"bot"}, {:app=>"Applebot", :match=>{"regex"=>"^Applebot/"}, :platform=>"bot"}, {:app=>"Archive.org", :match=>{"regex"=>"archive\\.org_bot"}, :platform=>"bot"}, {:app=>"atheerfm", :match=>{"regex"=>"^atheerfm/"}, :platform=>"bot"}, {:app=>"Audiomack", :match=>{"regex"=>"^Audiomack Podcast Processor/"}, :platform=>"bot"}, {:app=>"AudioWave feed parser", :match=>{"regex"=>"^AudioWaveBot/1\\.0"}, :platform=>"bot"}, {:app=>"AwarioSmartBot", :match=>{"regex"=>"^AwarioSmartBot/"}, :platform=>"bot"}, {:app=>"Babbar", :match=>{"regex"=>"Barkrowler/"}, :platform=>"bot"}, {:app=>"Baidu", :match=>{"regex"=>"\\(ce\\.baidu\\.com"}, :platform=>"bot"}, {:app=>"bbot", :match=>{"regex"=>"^bbot/"}, :platform=>"bot"}, {:app=>"Beacon Podcast Sync", :match=>{"regex"=>"^Beacon/\\d"}, :platform=>"bot"}, {:app=>"British Library", :match=>{"regex"=>"^bl\\.uk_ldfc_bot"}, :platform=>"bot"}, {:app=>"Blubrry Migration Service", :match=>{"regex"=>"^Blubrry Migration Service"}, :platform=>"bot"}, {:app=>"Bullhorn Crawler", :match=>{"regex"=>"^BullhornCrawler/"}, :platform=>"bot"}, {:app=>"Buzzsprout Importer", :match=>{"regex"=>"^Buzzsprout Importer"}, :platform=>"bot"}, {:app=>"Bytespider.*bytedance", :match=>{"regex"=>"Bytespider"}, :platform=>"bot"}, {:app=>"CastFeedValidator", :match=>{"regex"=>"^CastFeedValidator/"}, :platform=>"bot"}, {:app=>"Castopod", :match=>{"regex"=>"Castopod/\\d"}, :platform=>"bot"}, {:app=>"Chartable", :match=>{"regex"=>"^Trackable/"}, :platform=>"bot"}, {:app=>"Clark-Crawler, unknown", :match=>{"regex"=>"^clark-crawler2"}, :platform=>"bot"}, {:app=>"Claudebot", :match=>{"regex"=>"ClaudeBot/"}, :platform=>"bot"}, {:app=>"Critical Mention", :match=>{"regex"=>"^Podcast-CriticalMention/"}, :platform=>"bot"}, {:app=>"curl", :match=>{"regex"=>"^curl|^libcurl/|^PycURL/| curl/"}, :platform=>"bot"}, {:app=>"DataforSEO", :match=>{"regex"=>"DataForSeoBot/"}, :platform=>"bot"}, {:app=>"Datagnion Bot", :match=>{"regex"=>"^datagnionbot"}, :platform=>"bot"}, {:app=>"dataprovider.com", :match=>{"regex"=>"Dataprovider\\.com"}, :platform=>"bot"}, {:app=>"Daum", :match=>{"regex"=>"http://cs\\.daum\\.net"}, :platform=>"bot"}, {:app=>"Deepcast Podcast Sync", :match=>{"regex"=>"^DeepCast/\\d.*Sync"}, :platform=>"bot"}, {:app=>"Deezer Podcasters", :match=>{"regex"=>"^Deezer Podcasters/1\\.0"}, :platform=>"bot"}, {:app=>"Digg", :match=>{"regex"=>"^Digg "}, :platform=>"bot"}, {:app=>"dorada", :match=>{"regex"=>"support@dorada\\.co\\.uk"}, :platform=>"bot"}, {:app=>"DotBot", :match=>{"regex"=>"DotBot"}, :platform=>"bot"}, {:app=>"Downcast Bot", :match=>{"regex"=>"downcast feed consumer/"}, :platform=>"bot"}, {:app=>"DuckDuckBot", :match=>{"regex"=>"DuckDuckBot"}, :platform=>"bot"}, {:app=>"F-Secure Riddler", :match=>{"regex"=>"^Riddler "}, :platform=>"bot"}, {:app=>"Facebook Bot", :match=>{"regex"=>"FacebookBot|f?acebookexternalhit/?|^podcastbot$|Facebot|facebookexternalua|^facebookplatform/"}, :platform=>"bot"}, {:app=>"Feedly", :match=>{"regex"=>"^Feedly/"}, :platform=>"bot"}, {:app=>"Feedspot Fetcher", :match=>{"regex"=>"https://www\\.feedspot\\.com/fs/fetcher"}, :platform=>"bot"}, {:app=>"FlexGet", :match=>{"regex"=>"^FlexGet/"}, :platform=>"bot"}, {:app=>"Flipboard Proxy", :match=>{"regex"=>"FlipboardProxy/"}, :platform=>"bot"}, {:app=>"Fyyd", :match=>{"regex"=>"^fyyd-poll"}, :platform=>"bot"}, {:app=>"Go Storage Gateway V1", :match=>{"regex"=>"^storagegw-v1-go$"}, :platform=>"bot"}, {:app=>"Goodpods Bot", :match=>{"regex"=>"Goodpods/\\d+\\.\\d+"}, :platform=>"bot"}, {:app=>"Google Adsense Bot", :match=>{"regex"=>"Mediapartners-Google"}, :platform=>"bot"}, {:app=>"Google AdsBot", :match=>{"regex"=>"AdsBot-Google"}, :platform=>"bot"}, {:app=>"Google Podcasts Manager", :match=>{"regex"=>"Google-Podcast"}, :platform=>"bot"}, {:app=>"Googlebot", :match=>{"regex"=>"Googlebot/|Googlebot-Video/|Googlebot-Image/|^Feedfetcher-Google|google-xrawler|^Googlebot-News|^Google-Safety$"}, :platform=>"bot"}, {:app=>"Gumball", :match=>{"regex"=>"^Gumball"}, :platform=>"bot"}, {:app=>"Headliner", :match=>{"regex"=>"^Headliner/.*\\+https://headliner\\.app$"}, :platform=>"bot"}, {:app=>"HubSpot Crawler", :match=>{"regex"=>"HubSpot Crawler"}, :platform=>"bot"}, {:app=>"Internet Archive", :match=>{"regex"=>"Archive-It;|web\\.archive\\.org"}, :platform=>"bot"}, {:app=>"iOS share menu", :match=>{"regex"=>"^SharingUIService/"}, :platform=>"bot"}, {:app=>"IPFS Podcasting Node", :match=>{"regex"=>"^IPFS Podcasting Node/"}, :platform=>"bot"}, {:app=>"Jaunt", :match=>{"regex"=>"^Jaunt/"}, :platform=>"bot"}, {:app=>"l'Institut national de l'audiovisuel", :match=>{"regex"=>"INA dlweb"}, :platform=>"bot"}, {:app=>"Library and Archives Canada", :match=>{"regex"=>"LAC_IAHarvester/"}, :platform=>"bot"}, {:app=>"Libsyn", :match=>{"regex"=>"^Libsyn4"}, :platform=>"bot"}, {:app=>"libwww-perl", :match=>{"regex"=>"^libwww-perl| libwww-perl"}, :platform=>"bot"}, {:app=>"Livelap Crawler", :match=>{"regex"=>"LivelapBot"}, :platform=>"bot"}, {:app=>"ListenNotes", :match=>{"regex"=>"^ListenNotes/3\\.0 \\("}, :platform=>"bot"}, {:app=>"LTX71", :match=>{"regex"=>"^ltx71 "}, :platform=>"bot"}, {:app=>"MacOS share menu", :match=>{"regex"=>"^ShareSheetUI/"}, :platform=>"bot"}, {:app=>"MauiBot", :match=>{"regex"=>"^MauiBot"}, :platform=>"bot"}, {:app=>"Mastodon Bot", :match=>{"regex"=>"rb/.*Mastodon/"}, :platform=>"bot"}, {:app=>"Metacast Crawler", :match=>{"regex"=>"^Metacast/.* Crawler"}, :platform=>"bot"}, {:app=>"Microsoft Bingbot", :match=>{"regex"=>"(BingPreview/|adidxbot/|[bB]ingbot/)"}, :platform=>"bot"}, {:app=>"Microsoft Office", :match=>{"regex"=>"ms-office; MSOffice"}, :platform=>"bot"}, {:app=>"Mixcloud Podcast Importer", :match=>{"regex"=>"^MixcloudPodcastImporter/"}, :platform=>"bot"}, {:app=>"MJ12bot", :match=>{"regex"=>".*MJ12bot"}, :platform=>"bot"}, {:app=>"Mozilla Bot", :match=>{"regex"=>"^'?Mozilla(/5\\.0(\\.\\.\\.)?)?$|^\\(Mozilla/5\\.0\\)$"}, :platform=>"bot"}, {:app=>"MSN Bot", :match=>{"regex"=>"^msnbot/"}, :platform=>"bot"}, {:app=>"Neevabot", :match=>{"regex"=>".*Neevabot"}, :platform=>"bot"}, {:app=>"Netcraft Survey Agent", :match=>{"regex"=>" NetcraftSurveyAgent/"}, :platform=>"bot"}, {:app=>"Newspaper", :match=>{"regex"=>"^newspaper/\\d"}, :platform=>"bot"}, {:app=>"OgScrper", :match=>{"regex"=>"OgScrper"}, :platform=>"bot"}, {:app=>"OkDownload", :match=>{"regex"=>"^OkDownload/"}, :platform=>"bot"}, {:app=>"OP3 Fetcher", :match=>{"regex"=>"^op3-fetcher/"}, :platform=>"bot"}, {:app=>"OpenAI GPTBot", :match=>{"regex"=>"GPTBot/"}, :platform=>"bot"}, {:app=>"Overcast feed parser", :match=>{"regex"=>"^Overcast/1\\.0 Podcast Sync"}, :platform=>"bot"}, {:app=>"Pandora RSS crawler", :match=>{"regex"=>"^PandoraRSSCrawler"}, :platform=>"bot"}, {:app=>"PaperLi", :match=>{"regex"=>"PaperLiBot/"}, :platform=>"bot"}, {:app=>"PetalBot", :match=>{"regex"=>"PetalBot"}, :platform=>"bot"}, {:app=>"Pingdom", :match=>{"regex"=>"^Pingdom"}, :platform=>"bot"}, {:app=>"PlayerFM Podcast Sync", :match=>{"regex"=>"PlayerFM/.* Podcast Sync"}, :platform=>"bot"}, {:app=>"Podbean Importer", :match=>{"regex"=>"^Podbean Importer"}, :platform=>"bot"}, {:app=>"Podcastindex.org", :match=>{"regex"=>"^Podcastindex\\.org/"}, :platform=>"bot"}, {:app=>"Podcast de facto Standard", :match=>{"regex"=>"^PodcastStandard/"}, :platform=>"bot"}, {:app=>"Podcast Archiver", :match=>{"regex"=>"^Podcast%20Archiver/"}, :platform=>"bot"}, {:app=>"PodCloud", :match=>{"regex"=>"podCloud"}, :platform=>"bot"}, {:app=>"Podcorn", :match=>{"regex"=>"Podcorn/"}, :platform=>"bot"}, {:app=>"PodderBot", :match=>{"regex"=>"PodderBot/"}, :platform=>"bot"}, {:app=>"Podfollow", :match=>{"regex"=>"podfollowbot/"}, :platform=>"bot"}, {:app=>"Podgrab", :match=>{"regex"=>"^Podgrab$"}, :platform=>"bot"}, {:app=>"Podhound", :match=>{"regex"=>"PodhoundBeta"}, :platform=>"bot"}, {:app=>"Podio Bot", :match=>{"regex"=>"^Podio/"}, :platform=>"bot"}, {:app=>"Podnews", :match=>{"regex"=>"PodnewsBot"}, :platform=>"bot"}, {:app=>"Podnods Bot", :match=>{"regex"=>"(podnods-crawler|podnods)"}, :platform=>"bot"}, {:app=>"Podplay Podcast Sync", :match=>{"regex"=>"^Podplay-Podcast-Sync/"}, :platform=>"bot"}, {:app=>"Podroll Analyzer", :match=>{"regex"=>"^pdrl\\.fm "}, :platform=>"bot"}, {:app=>"Podscan Bot", :match=>{"regex"=>"^PodscanBot/"}, :platform=>"bot"}, {:app=>"Podscribe", :match=>{"regex"=>"(^Adswizz-podscribe/|^Podscribe/)"}, :platform=>"bot"}, {:app=>"PodUptime", :match=>{"regex"=>"(PodUptime|PodUptimeBot)/\\d"}, :platform=>"bot"}, {:app=>"Podverse Feed Parser", :match=>{"regex"=>"^Podverse/Feed Parser"}, :platform=>"bot"}, {:app=>"PodvineBot", :match=>{"regex"=>"^PodvineBot/"}, :platform=>"bot"}, {:app=>"PostRank Bot", :match=>{"regex"=>"^PostRank/"}, :platform=>"bot"}, {:app=>"PodwatchPro", :match=>{"regex"=>"Podwatch-Pro Crawler"}, :platform=>"bot"}, {:app=>"Puppeteer", :match=>{"regex"=>" HeadlessChrome/\\d"}, :platform=>"bot"}, {:app=>"python-requests", :match=>{"regex"=>"python-requests"}, :platform=>"bot"}, {:app=>"Qiniu spider", :match=>{"regex"=>"qiniu.*spider"}, :platform=>"bot"}, {:app=>"Radioline Stream Monitoring", :match=>{"regex"=>"^Radioline Stream Monitoring"}, :platform=>"bot"}, {:app=>"RedCircle", :match=>{"regex"=>"RedCircle"}, :platform=>"bot"}, {:app=>"Rephonic", :match=>{"regex"=>"^Rephonic/\\d"}, :platform=>"bot"}, {:app=>"Ridder Bot", :match=>{"regex"=>" RidderBot/"}, :platform=>"bot"}, {:app=>"RSS to Telegram", :match=>{"regex"=>"^RSStT"}, :platform=>"bot"}, {:app=>"RSSRadio", :match=>{"regex"=>"^RSSRadio \\("}, :platform=>"bot"}, {:app=>"Ruby Mechanize", :match=>{"regex"=>"^Mechanize|[ -]Mechanize/"}, :platform=>"bot"}, {:app=>"Screaming Frog SEO Spider", :match=>{"regex"=>"^Screaming Frog "}, :platform=>"bot"}, {:app=>"SearchAtlas.com SEO Crawler", :match=>{"regex"=>"^SearchAtlas.*Crawler"}, :platform=>"bot"}, {:app=>"SEMrushBot", :match=>{"regex"=>"SemrushBot/|^SEMrushBot$"}, :platform=>"bot"}, {:app=>"Serendeputy", :match=>{"regex"=>"SerendeputyBot/"}, :platform=>"bot"}, {:app=>"Simplecast", :match=>{"regex"=>"^Simplecast$"}, :platform=>"bot"}, {:app=>"Slack Bot", :match=>{"regex"=>"^Slackbot 1\\.0"}, :platform=>"bot"}, {:app=>"Snapchat Bot", :match=>{"regex"=>"://developers\\.snap\\.com/robots"}, :platform=>"bot"}, {:app=>"SoundOn Bot", :match=>{"regex"=>"^SoundOn/[\\d.]+\\s+\\(bot"}, :platform=>"bot"}, {:app=>"Speechmatics ASR", :match=>{"regex"=>"^Speechmatics-API/"}, :platform=>"bot"}, {:app=>"Spotify cache service", :match=>{"regex"=>"^Spotify/1\\.0$"}, :platform=>"bot"}, {:app=>"Stitcher Bot", :match=>{"regex"=>"^StitcherBot"}, :platform=>"bot"}, {:app=>"Substack Content Fetcher", :match=>{"regex"=>"^SubstackContentFetch/"}, :platform=>"bot"}, {:app=>"SupportingCast", :match=>{"regex"=>"^SupportingCast(/.*)?$"}, :platform=>"bot"}, {:app=>"Timpi search crawler", :match=>{"regex"=>"Timpibot/"}, :platform=>"bot"}, {:app=>"Tiny Tiny RSS", :match=>{"regex"=>"^Tiny Tiny RSS/"}, :platform=>"bot"}, {:app=>"Transistor", :match=>{"regex"=>"^Transistor\\.fm/"}, :platform=>"bot"}, {:app=>"Trendsmap Resolver", :match=>{"regex"=>"TrendsmapResolver/"}, :platform=>"bot"}, {:app=>"Triton Digital Podcast Sync", :match=>{"regex"=>"^Triton Digital Podcast Sync$"}, :platform=>"bot"}, {:app=>"Twingly Bot", :match=>{"regex"=>"Twingly Recon;"}, :platform=>"bot"}, {:app=>"Twitterbot", :match=>{"regex"=>"^Twitterbot"}, :platform=>"bot"}, {:app=>"Typhoeus", :match=>{"regex"=>"^Typhoeus"}, :platform=>"bot"}, {:app=>"UCast", :match=>{"regex"=>"^UCast/"}, :platform=>"bot"}, {:app=>"UptimeRobot", :match=>{"regex"=>"UptimeRobot/\\d"}, :platform=>"bot"}, {:app=>"TelegramBot", :match=>{"regex"=>"^TelegramBot "}, :platform=>"bot"}, {:app=>"theTradeDesk Content Web Scraper", :match=>{"regex"=>"TTD-Content"}, :platform=>"bot"}, {:app=>"Veritone Engine Toolkit", :match=>{"regex"=>"^veritone/engine-toolkit"}, :platform=>"bot"}, {:app=>"Vurbl", :match=>{"regex"=>"VurblBot"}, :platform=>"bot"}, {:app=>"Wget", :match=>{"regex"=>"Wget"}, :platform=>"bot"}, {:app=>"weborama", :match=>{"regex"=>"^weborama-fetcher"}, :platform=>"bot"}, {:app=>"Windows Crawler", :match=>{"regex"=>"^ZDM/.*Windows"}, :platform=>"bot"}, {:app=>"Windows PowerShell", :match=>{"regex"=>"WindowsPowerShell/"}, :platform=>"bot"}, {:app=>"WordPress", :match=>{"regex"=>"^WordPress"}, :platform=>"bot"}, {:app=>"wsrv.nl", :match=>{"regex"=>"https?://wsrv.nl/"}, :platform=>"bot"}, {:app=>"YaCy", :match=>{"regex"=>"^yacybot"}, :platform=>"bot"}, {:app=>"Yahoo Crawler", :match=>{"regex"=>"Yahoo! Slurp"}, :platform=>"bot"}, {:app=>"YandexBot", :match=>{"regex"=>"YandexBot/"}, :platform=>"bot"}, {:app=>"Zapier", :match=>{"regex"=>"^Zapier$"}, :platform=>"bot"}, {:app=>"Zencast", :match=>{"regex"=>"^Zencastr/"}, :platform=>"bot"}].freeze \ No newline at end of file diff --git a/lib/pod_ident/detection_rules_custom_bots.rb b/lib/pod_ident/detection_rules_custom_bots.rb index 136e266..4c53e38 100644 --- a/lib/pod_ident/detection_rules_custom_bots.rb +++ b/lib/pod_ident/detection_rules_custom_bots.rb @@ -1,3 +1,3 @@ # DO NOT EDIT THIS FILE - it gets automatically generated by running "bin/parse-rules" -CUSTOM_BOTS_RULES = [{:app=>"curl", :match=>{"regex"=>"^Faraday"}, :platform=>"bot"}, {:app=>"curl", :match=>{"regex"=>"like curl"}, :platform=>"bot"}].freeze \ No newline at end of file +CUSTOM_BOTS_RULES = [{:app=>"Apache HTTPClient", :match=>{"regex"=>"^Apache\\-HttpClient"}, :platform=>"bot"}, {:app=>"curl", :match=>{"regex"=>"^Faraday"}, :platform=>"bot"}, {:app=>"curl", :match=>{"regex"=>"like curl"}, :platform=>"bot"}].freeze \ No newline at end of file diff --git a/lib/pod_ident/version.rb b/lib/pod_ident/version.rb index 78c4d43..8d1d093 100644 --- a/lib/pod_ident/version.rb +++ b/lib/pod_ident/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module PodIdent - VERSION = '1.1.5' + VERSION = '1.1.6' end diff --git a/spec/detection_rules_bots.rb b/spec/detection_rules_bots.rb index 505cc0a..8505a53 100644 --- a/spec/detection_rules_bots.rb +++ b/spec/detection_rules_bots.rb @@ -1,3 +1,3 @@ # DO NOT EDIT THIS FILE - it gets automatically generated by running "bin/parse-rules" -BOTS_RULES = [{:app=>"AAABot - unknown bot", :match=>{"regex"=>"AAABot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AAABot"}]}}, {:app=>"Adswizz", :match=>{"regex"=>"^AIS VirtualListener"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AIS VirtualListener"}]}}, {:app=>"AhrefsBot", :match=>{"regex"=>"AhrefsBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; AhrefsBot/7.0; http://ahrefs.com/robot/)"}, {"userAgent"=>"Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)"}]}}, {:app=>"AirableBot", :match=>{"regex"=>"AirableBot-Podcast/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AirableBot-Podcast/1.0 (+https//www.airablenow.com)"}, {"userAgent"=>"AirableBot-Podcast/1.0 ( https//www.airablenow.com)"}]}}, {:app=>"Alexa Flash Briefing cache", :match=>{"regex"=>"^AmazonNewsContentService"}, :platform=>"bot"}, {:app=>"AlignaBot", :match=>{"regex"=>"^Alignabot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Alignabot 1.0 (https://www.alignable.com/)"}]}}, {:app=>"Amazon Music Podcasts Bot", :match=>{"regex"=>"^Amazon Music Podcast"}, :platform=>"bot"}, {:app=>"Anchor Importer", :match=>{"regex"=>"AnchorImport"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AnchorImport/1.0"}]}}, {:app=>"Apache HTTPClient", :match=>{"regex"=>"^Apache\\-HttpClient"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Apache-HttpClient/5.1.4 (Java/1.8.0_144)"}, {"userAgent"=>"Apache-HttpClient/5.1.4 (Java/1.8.0_91)"}]}}, {:app=>"Apple Podcasts (Watch)", :match=>{"regex"=>"^atc/|\\(null\\) watchOS/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"atc/1.0"}, {"userAgent"=>"atc/1.0 watchOS/6.2 model/Watch3,3 hwp/t8004 build/17T529 (6; dt:155)"}, {"userAgent"=>"atc/1.0 watchOS/6.2.8 model/Watch2,3 hwp/t8002 build/17U63 (6; dt:133)"}, {"userAgent"=>"atc/1.0 watchOS/6.2.8 model/Watch3,3 hwp/t8004 build/17U63 (6; dt:155)"}, {"userAgent"=>"atc/1.0 watchOS/6.2.8 model/Watch4,2 hwp/t8006 build/17U63 (6; dt:191)"}, {"userAgent"=>"atc/1.0 watchOS/7.0.2 model/Watch5,10 hwp/t8006 build/18R402 (6; dt:233)"}, {"userAgent"=>"atc/1.0 watchOS/7.0.2 model/Watch5,11 hwp/t8006 build/18R402 (6; dt:234)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch4,2 hwp/t8006 build/18R590 (6; dt:191)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch4,3 hwp/t8006 build/18R590 (6; dt:192)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch4,4 hwp/t8006 build/18R590 (6; dt:193)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch5,1 hwp/t8006 build/18R590 (6; dt:201)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch5,3 hwp/t8006 build/18R590 (6; dt:202)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch5,4 hwp/t8006 build/18R590 (6; dt:202)"}, {"userAgent"=>"(null)/(null) watchOS/5.0.1 model/Watch3,3 hwp/t8004 build/16R381 (6; dt:155)"}]}}, {:app=>"Apple Podcasts automated checks", :match=>{"regex"=>"(iTMS|itunesstored|itms)"}, :platform=>"bot"}, {:app=>"Applebot", :match=>{"regex"=>"^Applebot/"}, :platform=>"bot"}, {:app=>"Archive.org", :match=>{"regex"=>"archive\\.org_bot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; archive.org_bot http://archive.org/details/archive.org_bot)"}]}}, {:app=>"atheerfm", :match=>{"regex"=>"^atheerfm/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"atheerfm/1 CFNetwork/758.3.15 Darwin/15.3.0"}]}}, {:app=>"Audiomack", :match=>{"regex"=>"^Audiomack Podcast Processor/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Audiomack Podcast Processor/1.0 (https://audiomack.com/)"}]}}, {:app=>"AudioWave feed parser", :match=>{"regex"=>"^AudioWaveBot/1\\.0"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AudioWaveBot/1.0"}]}}, {:app=>"AwarioSmartBot", :match=>{"regex"=>"^AwarioSmartBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)"}]}}, {:app=>"Babbar", :match=>{"regex"=>"Barkrowler/"}, :platform=>"bot"}, {:app=>"Baidu", :match=>{"regex"=>"\\(ce\\.baidu\\.com"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Baidu-YunGuanCe-SLABot(ce.baidu.com)"}]}}, {:app=>"bbot", :match=>{"regex"=>"^bbot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"bbot/0.1"}]}}, {:app=>"British Library", :match=>{"regex"=>"^bl\\.uk_ldfc_bot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"bl.uk_ldfc_bot/3.4.0-20200518 ( http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)"}]}}, {:app=>"Blubrry Migration Service", :match=>{"regex"=>"^Blubrry Migration Service"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Blubrry Migration Service"}]}}, {:app=>"Buzzsprout Importer", :match=>{"regex"=>"^Buzzsprout Importer"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Buzzsprout Importer"}]}}, {:app=>"CastFeedValidator", :match=>{"regex"=>"^CastFeedValidator/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"CastFeedValidator/3.0.5 (https://castfeedvalidator.com)"}]}}, {:app=>"Castopod", :match=>{"regex"=>"Castopod/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Castopod/1.0"}, {"userAgent"=>"Castopod/1.1.2"}]}}, {:app=>"Chartable", :match=>{"regex"=>"^Trackable/"}, :platform=>"bot"}, {:app=>"Clark-Crawler, unknown", :match=>{"regex"=>"^clark-crawler2"}, :platform=>"bot"}, {:app=>"Critical Mention", :match=>{"regex"=>"^Podcast-CriticalMention/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcast-CriticalMention/1.0"}]}}, {:app=>"curl", :match=>{"regex"=>"^curl|^libcurl/|^PycURL/| curl/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"libcurl/7.65.1"}, {"userAgent"=>"PycURL/7.43.0 libcurl/7.47.0 GnuTLS/3.4.10 zlib/1.2.8 libidn/1.32 librtmp/2.3"}, {"userAgent"=>"GuzzleHttp/6.1.0 curl/7.50.1 PHP/7.0.13-0ubuntu0.16.10.1"}]}}, {:app=>"DataforSEO", :match=>{"regex"=>"DataForSeoBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)"}]}}, {:app=>"Datagnion Bot", :match=>{"regex"=>"^datagnionbot"}, :platform=>"bot"}, {:app=>"dataprovider.com", :match=>{"regex"=>"Dataprovider\\.com"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Dataprovider.com)"}]}}, {:app=>"Daum", :match=>{"regex"=>"http://cs\\.daum\\.net"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)"}]}}, {:app=>"Deezer Podcasters", :match=>{"regex"=>"^Deezer Podcasters/1\\.0"}, :platform=>"bot"}, {:app=>"Digg", :match=>{"regex"=>"^Digg "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Digg Mozilla/5.0 (Digg/1.0; support@digg.com)"}]}}, {:app=>"dorada", :match=>{"regex"=>"support@dorada\\.co\\.uk"}, :platform=>"bot"}, {:app=>"DotBot", :match=>{"regex"=>"DotBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)"}, {"userAgent"=>"Mozilla/5.0 (compatible; DotBot/1.2; https://opensiteexplorer.org/dotbot; help@moz.com)"}]}}, {:app=>"Downcast Bot", :match=>{"regex"=>"downcast feed consumer/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"downcast feed consumer/0.0.175; (mode=dev; id=u2NgjBSPM6; downcast.fm)"}]}}, {:app=>"DuckDuckBot", :match=>{"regex"=>"DuckDuckBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)"}]}}, {:app=>"F-Secure Riddler", :match=>{"regex"=>"^Riddler "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Riddler (http://riddler.io/about)"}]}}, {:app=>"Facebook Bot", :match=>{"regex"=>"FacebookBot|f?acebookexternalhit/?|^podcastbot$|Facebot|facebookexternalua|^facebookplatform/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"facebookexternalhit/1.1 ( http://www.facebook.com/externalhit_uatext.php)"}, {"userAgent"=>"podcastbot"}, {"userAgent"=>"facebookexternalhit"}, {"userAgent"=>"Facebot"}, {"userAgent"=>"facebookexternalua"}, {"userAgent"=>"facebookplatform/1.0 (+http://developers.facebook.com)"}, {"userAgent"=>"acebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)"}]}}, {:app=>"Feedly", :match=>{"regex"=>"^Feedly/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)"}]}}, {:app=>"Feedspot Fetcher", :match=>{"regex"=>"https://www\\.feedspot\\.com/fs/fetcher"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)"}]}}, {:app=>"FlexGet", :match=>{"regex"=>"^FlexGet/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"FlexGet/3.3.4 (www.flexget.com)"}]}}, {:app=>"Flipboard Proxy", :match=>{"regex"=>"FlipboardProxy/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)"}]}}, {:app=>"Fyyd", :match=>{"regex"=>"^fyyd-poll"}, :platform=>"bot"}, {:app=>"Go Storage Gateway V1", :match=>{"regex"=>"^storagegw-v1-go$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"storagegw-v1-go"}]}}, {:app=>"Goodpods Bot", :match=>{"regex"=>"Goodpods/\\d+\\.\\d+"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Goodpods/2.2"}]}}, {:app=>"Google Adsense Bot", :match=>{"regex"=>"Mediapartners-Google"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mediapartners-Google"}]}}, {:app=>"Google AdsBot", :match=>{"regex"=>"AdsBot-Google"}, :platform=>"bot"}, {:app=>"Google Podcasts Manager", :match=>{"regex"=>"Google-Podcast"}, :platform=>"bot"}, {:app=>"Googlebot", :match=>{"regex"=>"Googlebot/|Googlebot-Video/|Googlebot-Image/|^Feedfetcher-Google|google-xrawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; http://www.google.com/bot.html)"}, {"userAgent"=>"Googlebot-Image/1.0"}, {"userAgent"=>"Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=4653447469100830145)"}, {"userAgent"=>"google-xrawler"}]}}, {:app=>"Gumball", :match=>{"regex"=>"^Gumball"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Gumball.fm Analytics Prefix Checker"}]}}, {:app=>"Headliner", :match=>{"regex"=>"Headliner/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Headliner/1.0.0 +https://headliner.app"}]}}, {:app=>"HubSpot Crawler", :match=>{"regex"=>"HubSpot Crawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; HubSpot Crawler; +https://www.hubspot.com)"}]}}, {:app=>"Internet Archive", :match=>{"regex"=>"Archive-It;|web\\.archive\\.org"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; special_archiver; Archive-It; http://archive-it.org/files/site-owners-special.html)"}, {"userAgent"=>"ia_archiver-web.archive.org"}]}}, {:app=>"Jaunt", :match=>{"regex"=>"^Jaunt/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Jaunt/1.5"}]}}, {:app=>"l'Institut national de l'audiovisuel", :match=>{"regex"=>"INA dlweb"}, :platform=>"bot"}, {:app=>"Libsyn", :match=>{"regex"=>"^Libsyn4"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Libsyn4-Download"}]}}, {:app=>"libwww-perl", :match=>{"regex"=>"^libwww-perl| libwww-perl"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"LWP::Simple/6.34 libwww-perl/6.34"}]}}, {:app=>"Livelap Crawler", :match=>{"regex"=>"LivelapBot"}, :platform=>"bot"}, {:app=>"LTX71", :match=>{"regex"=>"^ltx71 "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"ltx71 - (http://ltx71.com/)"}]}}, {:app=>"MauiBot", :match=>{"regex"=>"^MauiBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"MauiBot (crawler.feedback dc@gmail.com)"}]}}, {:app=>"Mastodon Bot", :match=>{"regex"=>"rb/.*Mastodon/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"http.rb/5.0.4 (Mastodon/3.5.3; +https://mastodon.xyz/) Bot"}]}}, {:app=>"Microsoft Bingbot", :match=>{"regex"=>"(BingPreview/|adidxbot/|[bB]ingbot/)"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534 (KHTML, like Gecko) BingPreview/1.0b"}, {"userAgent"=>"Mozilla/5.0 (compatible; Bingbot/2.0; +http://www.bing.com/bingbot.htm)"}]}}, {:app=>"Microsoft Office", :match=>{"regex"=>"ms-office; MSOffice"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/4.0 (compatible; ms-office; MSOffice 16)"}]}}, {:app=>"Mixcloud Podcast Importer", :match=>{"regex"=>"^MixcloudPodcastImporter/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"MixcloudPodcastImporter/2.0"}]}}, {:app=>"MJ12bot", :match=>{"regex"=>".*MJ12bot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)"}]}}, {:app=>"Mozilla Bot", :match=>{"regex"=>"^'?Mozilla(/5\\.0(\\.\\.\\.)?)?$|^\\(Mozilla/5\\.0\\)$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0"}, {"userAgent"=>"Mozilla"}, {"userAgent"=>"Mozilla/5.0..."}, {"userAgent"=>"'Mozilla/5.0"}, {"userAgent"=>"(Mozilla/5.0)"}]}}, {:app=>"MSN Bot", :match=>{"regex"=>"^msnbot/"}, :platform=>"bot"}, {:app=>"Neevabot", :match=>{"regex"=>".*Neevabot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Neevabot/1.0; https://neeva.com/neevabot)"}]}}, {:app=>"Netcraft Survey Agent", :match=>{"regex"=>" NetcraftSurveyAgent/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)"}]}}, {:app=>"Newspaper", :match=>{"regex"=>"^newspaper/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"newspaper/0.3.0"}]}}, {:app=>"OgScrper", :match=>{"regex"=>"OgScrper"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"OgScrper"}]}}, {:app=>"OkDownload", :match=>{"regex"=>"^OkDownload/"}, :platform=>"bot"}, {:app=>"OP3 Fetcher", :match=>{"regex"=>"^op3-fetcher/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"op3-fetcher/1.0 (https://op3.dev)"}]}}, {:app=>"Overcast feed parser", :match=>{"regex"=>"^Overcast/1\\.0 Podcast Sync"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Overcast/1.0 Podcast Sync"}]}}, {:app=>"Pandora RSS crawler", :match=>{"regex"=>"^PandoraRSSCrawler"}, :platform=>"bot"}, {:app=>"PaperLi", :match=>{"regex"=>"PaperLiBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)"}]}}, {:app=>"PetalBot", :match=>{"regex"=>"PetalBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot; https://aspiegel.com/petalbot)"}]}}, {:app=>"Pingdom", :match=>{"regex"=>"^Pingdom"}, :platform=>"bot"}, {:app=>"PlayerFM Podcast Sync", :match=>{"regex"=>"PlayerFM/.* Podcast Sync"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PlayerFM/1.0 Podcast Sync (0 subscribers; url=https://player.fm/series/series-2567872)"}]}}, {:app=>"Podbean Importer", :match=>{"regex"=>"^Podbean Importer"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podbean Importer"}]}}, {:app=>"Podcastindex.org", :match=>{"regex"=>"^Podcastindex\\.org/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcastindex.org/v0.3.3 (Aggrivate)"}]}}, {:app=>"Podcast de facto Standard", :match=>{"regex"=>"^PodcastStandard/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodcastStandard/0.1.0 (+https://podcast-standard.org)"}]}}, {:app=>"Podcast Archiver", :match=>{"regex"=>"^Podcast%20Archiver/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcast%20Archiver/3000000003 CFNetwork/1406.0.4 Darwin/22.4.0"}]}}, {:app=>"PodCloud", :match=>{"regex"=>"podCloud"}, :platform=>"bot"}, {:app=>"Podcorn", :match=>{"regex"=>"Podcorn/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcorn/1.0"}]}}, {:app=>"PodderBot", :match=>{"regex"=>"PodderBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodderBot/1.0"}]}}, {:app=>"Podfollow", :match=>{"regex"=>"podfollowbot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 https://podfollow.com/crawling podfollowbot/1.0"}]}}, {:app=>"Podgrab", :match=>{"regex"=>"^Podgrab$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podgrab"}]}}, {:app=>"Podhound", :match=>{"regex"=>"PodhoundBeta"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodhoundBeta"}]}}, {:app=>"Podio Bot", :match=>{"regex"=>"^Podio/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podio/1.0"}]}}, {:app=>"Podnews", :match=>{"regex"=>"PodnewsBot"}, :platform=>"bot"}, {:app=>"Podnods Bot", :match=>{"regex"=>"(podnods-crawler|podnods)"}, :platform=>"bot"}, {:app=>"Podscribe", :match=>{"regex"=>"(^Adswizz-podscribe/|^Podscribe/)"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Adswizz-podscribe/1.0"}, {"userAgent"=>"Podscribe/1.1"}]}}, {:app=>"Podverse Feed Parser", :match=>{"regex"=>"^Podverse/Feed Parser"}, :platform=>"bot"}, {:app=>"PodvineBot", :match=>{"regex"=>"^PodvineBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodvineBot/4.0.1 (www.podvine.com)"}]}}, {:app=>"PostRank Bot", :match=>{"regex"=>"^PostRank/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PostRank/2.0 (postrank.com; 1 subscribers)"}]}}, {:app=>"PodwatchPro", :match=>{"regex"=>"Podwatch-Pro Crawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podwatch-Pro Crawler"}]}}, {:app=>"Puppeteer", :match=>{"regex"=>" HeadlessChrome/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/105.0.5173.0 Safari/537.36"}]}}, {:app=>"python-requests", :match=>{"regex"=>"python-requests"}, :platform=>"bot"}, {:app=>"Qiniu spider", :match=>{"regex"=>"qiniu.*spider"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"z0-qiniu-imgstg-spider-1.0"}]}}, {:app=>"RedCircle", :match=>{"regex"=>"RedCircle"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"RedCircle"}]}}, {:app=>"Ridder Bot", :match=>{"regex"=>" RidderBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co)"}]}}, {:app=>"RSS to Telegram", :match=>{"regex"=>"^RSStT"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"RSStT/2.2.1 RSS Reader"}]}}, {:app=>"RSSRadio", :match=>{"regex"=>"^RSSRadio \\("}, :platform=>"bot"}, {:app=>"Ruby Mechanize", :match=>{"regex"=>"^Mechanize|[ -]Mechanize/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Mechanize/2.7.4)"}, {"userAgent"=>"WWW-Mechanize/1.72"}]}}, {:app=>"Screaming Frog SEO Spider", :match=>{"regex"=>"^Screaming Frog "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Screaming Frog SEO Spider/5.1"}]}}, {:app=>"SearchAtlas.com SEO Crawler", :match=>{"regex"=>"^SearchAtlas.*Crawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SearchAtlas.com SEO Crawler"}]}}, {:app=>"SEMrushBot", :match=>{"regex"=>"SemrushBot/|^SEMrushBot$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; SemrushBot/6~bl; http://www.semrush.com/bot.html)"}, {"userAgent"=>"SEMrushBot"}]}}, {:app=>"Serendeputy", :match=>{"regex"=>"SerendeputyBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)"}]}}, {:app=>"Simplecast", :match=>{"regex"=>"^Simplecast$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Simplecast"}]}}, {:app=>"Slack Bot", :match=>{"regex"=>"^Slackbot 1\\.0"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Slackbot 1.0 ( https://api.slack.com/robots)"}]}}, {:app=>"Snapchat Bot", :match=>{"regex"=>"://developers\\.snap\\.com/robots"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots"}]}}, {:app=>"SoundOn Bot", :match=>{"regex"=>"^SoundOn/[\\d.]+\\s+\\(bot"}, :platform=>"bot"}, {:app=>"Spotify cache service", :match=>{"regex"=>"^Spotify/1\\.0$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Spotify/1.0"}]}}, {:app=>"Stitcher Bot", :match=>{"regex"=>"^StitcherBot"}, :platform=>"bot"}, {:app=>"Substack Content Fetcher", :match=>{"regex"=>"^SubstackContentFetch/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SubstackContentFetch/1.0 (https://substack.com/)"}]}}, {:app=>"SupportingCast", :match=>{"regex"=>"^SupportingCast(/.*)?$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SupportingCast/1.0"}, {"userAgent"=>"SupportingCast"}]}}, {:app=>"Timpi search crawler", :match=>{"regex"=>"Timpibot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Timpibot/0.8 ( http://www.timpi.io)"}]}}, {:app=>"Tiny Tiny RSS", :match=>{"regex"=>"^Tiny Tiny RSS/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Tiny Tiny RSS/22.12-c30b24d09 (Unsupported) (https://tt-rss.org/)"}]}}, {:app=>"Trendsmap Resolver", :match=>{"regex"=>"TrendsmapResolver/"}, :platform=>"bot"}, {:app=>"Twingly Bot", :match=>{"regex"=>"Twingly Recon;"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Twingly Recon; twingly.com)"}]}}, {:app=>"Twitterbot", :match=>{"regex"=>"^Twitterbot"}, :platform=>"bot"}, {:app=>"Typhoeus", :match=>{"regex"=>"^Typhoeus"}, :platform=>"bot"}, {:app=>"UCast", :match=>{"regex"=>"^UCast/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"UCast/1.0 Podcast Sync (1 subscribers; feed-id=aHR0cHM6Ly9mZWVkcy5idXp6c3Byb3V0LmNvbS8yMDg0OTQucnNzEAEBAD32-0242-42AC-8583-21E9BAD8C544; +http://www.ucastapp.com/)"}]}}, {:app=>"TelegramBot", :match=>{"regex"=>"^TelegramBot "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"TelegramBot (like TwitterBot)"}]}}, {:app=>"theTradeDesk Content Web Scraper", :match=>{"regex"=>"TTD-Content"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; TTD-Content; +https://www.thetradedesk.com/general/ttd-content)"}]}}, {:app=>"Veritone Engine Toolkit", :match=>{"regex"=>"^veritone/engine-toolkit"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"veritone/engine-toolkit-3.0"}]}}, {:app=>"Vurbl", :match=>{"regex"=>"VurblBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 https://vurbl.com VurblBot/1.0"}]}}, {:app=>"Wget", :match=>{"regex"=>"Wget"}, :platform=>"bot"}, {:app=>"weborama", :match=>{"regex"=>"^weborama-fetcher"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"weborama-fetcher (+http://www.weborama.com)"}]}}, {:app=>"Windows Crawler", :match=>{"regex"=>"^ZDM/.*Windows"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"ZDM/4.0; Windows Mobile 7.0;"}]}}, {:app=>"Windows PowerShell", :match=>{"regex"=>"WindowsPowerShell/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Windows NT; Windows NT 10.0; en-US) WindowsPowerShell/5.1.19041.2673"}]}}, {:app=>"WordPress", :match=>{"regex"=>"^WordPress"}, :platform=>"bot"}, {:app=>"wsrv.nl", :match=>{"regex"=>"https?://wsrv.nl/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; ImageFetcher/9.0; +http://wsrv.nl/)"}]}}, {:app=>"YaCy", :match=>{"regex"=>"^yacybot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"yacybot (/global; amd64 Linux 5.9.8-zen1-1-zen; java 1.8.0_265; Europe/de) http://yacy.net/bot.html"}]}}, {:app=>"Yahoo Crawler", :match=>{"regex"=>"Yahoo! Slurp"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Crawler Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"}]}}, {:app=>"YandexBot", :match=>{"regex"=>"YandexBot/"}, :platform=>"bot"}, {:app=>"Zapier", :match=>{"regex"=>"^Zapier$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Zapier"}]}}, {:app=>"Zencast", :match=>{"regex"=>"^Zencastr/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Zencastr/2.0"}]}}].freeze \ No newline at end of file +BOTS_RULES = [{:app=>"AAABot - unknown bot", :match=>{"regex"=>"AAABot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AAABot"}]}}, {:app=>"Adswizz", :match=>{"regex"=>"^AIS VirtualListener"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AIS VirtualListener"}]}}, {:app=>"ma Podcast (agma) Bot", :match=>{"regex"=>"^agma/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"agma/1.0"}]}}, {:app=>"AhrefsBot", :match=>{"regex"=>"AhrefsBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; AhrefsBot/7.0; http://ahrefs.com/robot/)"}, {"userAgent"=>"Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)"}]}}, {:app=>"AirableBot", :match=>{"regex"=>"AirableBot-Podcast/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AirableBot-Podcast/1.0 (+https//www.airablenow.com)"}, {"userAgent"=>"AirableBot-Podcast/1.0 ( https//www.airablenow.com)"}]}}, {:app=>"Alexa Flash Briefing cache", :match=>{"regex"=>"^AmazonNewsContentService"}, :platform=>"bot"}, {:app=>"AlignaBot", :match=>{"regex"=>"^Alignabot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Alignabot 1.0 (https://www.alignable.com/)"}]}}, {:app=>"All Ears", :match=>{"regex"=>"^AllEars/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AllEars/1.0; (+www.allears.ai)"}]}}, {:app=>"Amazon Music Podcasts Bot", :match=>{"regex"=>"^Amazon Music Podcast"}, :platform=>"bot"}, {:app=>"Anchor Importer", :match=>{"regex"=>"AnchorImport"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AnchorImport/1.0"}]}}, {:app=>"Apple Notes share extension", :match=>{"regex"=>"^com.apple.(mobilenotes.SharingExtension|Notes.SharingExtension)/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"com.apple.mobilenotes.SharingExtension/2511 CFNetwork/1494.0.7 Darwin/23.4.0"}, {"userAgent"=>"com.apple.Notes.SharingExtension/2812 CFNetwork/1494.0.7 Darwin/23.4.0"}]}}, {:app=>"Apple Podcasts (Watch)", :match=>{"regex"=>"^atc/|\\(null\\) watchOS/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"atc/1.0"}, {"userAgent"=>"atc/1.0 watchOS/6.2 model/Watch3,3 hwp/t8004 build/17T529 (6; dt:155)"}, {"userAgent"=>"atc/1.0 watchOS/6.2.8 model/Watch2,3 hwp/t8002 build/17U63 (6; dt:133)"}, {"userAgent"=>"atc/1.0 watchOS/6.2.8 model/Watch3,3 hwp/t8004 build/17U63 (6; dt:155)"}, {"userAgent"=>"atc/1.0 watchOS/6.2.8 model/Watch4,2 hwp/t8006 build/17U63 (6; dt:191)"}, {"userAgent"=>"atc/1.0 watchOS/7.0.2 model/Watch5,10 hwp/t8006 build/18R402 (6; dt:233)"}, {"userAgent"=>"atc/1.0 watchOS/7.0.2 model/Watch5,11 hwp/t8006 build/18R402 (6; dt:234)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch4,2 hwp/t8006 build/18R590 (6; dt:191)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch4,3 hwp/t8006 build/18R590 (6; dt:192)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch4,4 hwp/t8006 build/18R590 (6; dt:193)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch5,1 hwp/t8006 build/18R590 (6; dt:201)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch5,3 hwp/t8006 build/18R590 (6; dt:202)"}, {"userAgent"=>"atc/1.0 watchOS/7.1 model/Watch5,4 hwp/t8006 build/18R590 (6; dt:202)"}, {"userAgent"=>"(null)/(null) watchOS/5.0.1 model/Watch3,3 hwp/t8004 build/16R381 (6; dt:155)"}]}}, {:app=>"Apple Podcasts automated checks", :match=>{"regex"=>"(iTMS|itunesstored|itms)"}, :platform=>"bot"}, {:app=>"Applebot", :match=>{"regex"=>"^Applebot/"}, :platform=>"bot"}, {:app=>"Archive.org", :match=>{"regex"=>"archive\\.org_bot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; archive.org_bot http://archive.org/details/archive.org_bot)"}]}}, {:app=>"atheerfm", :match=>{"regex"=>"^atheerfm/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"atheerfm/1 CFNetwork/758.3.15 Darwin/15.3.0"}]}}, {:app=>"Audiomack", :match=>{"regex"=>"^Audiomack Podcast Processor/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Audiomack Podcast Processor/1.0 (https://audiomack.com/)"}]}}, {:app=>"AudioWave feed parser", :match=>{"regex"=>"^AudioWaveBot/1\\.0"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AudioWaveBot/1.0"}]}}, {:app=>"AwarioSmartBot", :match=>{"regex"=>"^AwarioSmartBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)"}]}}, {:app=>"Babbar", :match=>{"regex"=>"Barkrowler/"}, :platform=>"bot"}, {:app=>"Baidu", :match=>{"regex"=>"\\(ce\\.baidu\\.com"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Baidu-YunGuanCe-SLABot(ce.baidu.com)"}]}}, {:app=>"bbot", :match=>{"regex"=>"^bbot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"bbot/0.1"}]}}, {:app=>"Beacon Podcast Sync", :match=>{"regex"=>"^Beacon/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Beacon/1.0 Podcast Sync +https://getbeacons.com/"}]}}, {:app=>"British Library", :match=>{"regex"=>"^bl\\.uk_ldfc_bot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"bl.uk_ldfc_bot/3.4.0-20200518 ( http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)"}]}}, {:app=>"Blubrry Migration Service", :match=>{"regex"=>"^Blubrry Migration Service"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Blubrry Migration Service"}]}}, {:app=>"Bullhorn Crawler", :match=>{"regex"=>"^BullhornCrawler/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"BullhornCrawler/unknown-version(+http://bullhorn.fm/)"}]}}, {:app=>"Buzzsprout Importer", :match=>{"regex"=>"^Buzzsprout Importer"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Buzzsprout Importer"}]}}, {:app=>"Bytespider.*bytedance", :match=>{"regex"=>"Bytespider"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)"}]}}, {:app=>"CastFeedValidator", :match=>{"regex"=>"^CastFeedValidator/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"CastFeedValidator/3.0.5 (https://castfeedvalidator.com)"}]}}, {:app=>"Castopod", :match=>{"regex"=>"Castopod/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Castopod/1.0"}, {"userAgent"=>"Castopod/1.1.2"}]}}, {:app=>"Chartable", :match=>{"regex"=>"^Trackable/"}, :platform=>"bot"}, {:app=>"Clark-Crawler, unknown", :match=>{"regex"=>"^clark-crawler2"}, :platform=>"bot"}, {:app=>"Claudebot", :match=>{"regex"=>"ClaudeBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)"}]}}, {:app=>"Critical Mention", :match=>{"regex"=>"^Podcast-CriticalMention/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcast-CriticalMention/1.0"}]}}, {:app=>"curl", :match=>{"regex"=>"^curl|^libcurl/|^PycURL/| curl/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"libcurl/7.65.1"}, {"userAgent"=>"PycURL/7.43.0 libcurl/7.47.0 GnuTLS/3.4.10 zlib/1.2.8 libidn/1.32 librtmp/2.3"}, {"userAgent"=>"GuzzleHttp/6.1.0 curl/7.50.1 PHP/7.0.13-0ubuntu0.16.10.1"}]}}, {:app=>"DataforSEO", :match=>{"regex"=>"DataForSeoBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)"}]}}, {:app=>"Datagnion Bot", :match=>{"regex"=>"^datagnionbot"}, :platform=>"bot"}, {:app=>"dataprovider.com", :match=>{"regex"=>"Dataprovider\\.com"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Dataprovider.com)"}]}}, {:app=>"Daum", :match=>{"regex"=>"http://cs\\.daum\\.net"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)"}]}}, {:app=>"Deepcast Podcast Sync", :match=>{"regex"=>"^DeepCast/\\d.*Sync"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"DeepCast/4.3.0 Podcast Sync (+https://deepcast.fm)"}]}}, {:app=>"Deezer Podcasters", :match=>{"regex"=>"^Deezer Podcasters/1\\.0"}, :platform=>"bot"}, {:app=>"Digg", :match=>{"regex"=>"^Digg "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Digg Mozilla/5.0 (Digg/1.0; support@digg.com)"}]}}, {:app=>"dorada", :match=>{"regex"=>"support@dorada\\.co\\.uk"}, :platform=>"bot"}, {:app=>"DotBot", :match=>{"regex"=>"DotBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)"}, {"userAgent"=>"Mozilla/5.0 (compatible; DotBot/1.2; https://opensiteexplorer.org/dotbot; help@moz.com)"}]}}, {:app=>"Downcast Bot", :match=>{"regex"=>"downcast feed consumer/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"downcast feed consumer/0.0.175; (mode=dev; id=u2NgjBSPM6; downcast.fm)"}]}}, {:app=>"DuckDuckBot", :match=>{"regex"=>"DuckDuckBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)"}]}}, {:app=>"F-Secure Riddler", :match=>{"regex"=>"^Riddler "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Riddler (http://riddler.io/about)"}]}}, {:app=>"Facebook Bot", :match=>{"regex"=>"FacebookBot|f?acebookexternalhit/?|^podcastbot$|Facebot|facebookexternalua|^facebookplatform/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"facebookexternalhit/1.1 ( http://www.facebook.com/externalhit_uatext.php)"}, {"userAgent"=>"podcastbot"}, {"userAgent"=>"facebookexternalhit"}, {"userAgent"=>"Facebot"}, {"userAgent"=>"facebookexternalua"}, {"userAgent"=>"facebookplatform/1.0 (+http://developers.facebook.com)"}, {"userAgent"=>"acebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)"}]}}, {:app=>"Feedly", :match=>{"regex"=>"^Feedly/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)"}]}}, {:app=>"Feedspot Fetcher", :match=>{"regex"=>"https://www\\.feedspot\\.com/fs/fetcher"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)"}]}}, {:app=>"FlexGet", :match=>{"regex"=>"^FlexGet/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"FlexGet/3.3.4 (www.flexget.com)"}]}}, {:app=>"Flipboard Proxy", :match=>{"regex"=>"FlipboardProxy/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)"}]}}, {:app=>"Fyyd", :match=>{"regex"=>"^fyyd-poll"}, :platform=>"bot"}, {:app=>"Go Storage Gateway V1", :match=>{"regex"=>"^storagegw-v1-go$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"storagegw-v1-go"}]}}, {:app=>"Goodpods Bot", :match=>{"regex"=>"Goodpods/\\d+\\.\\d+"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Goodpods/2.2"}]}}, {:app=>"Google Adsense Bot", :match=>{"regex"=>"Mediapartners-Google"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mediapartners-Google"}]}}, {:app=>"Google AdsBot", :match=>{"regex"=>"AdsBot-Google"}, :platform=>"bot"}, {:app=>"Google Podcasts Manager", :match=>{"regex"=>"Google-Podcast"}, :platform=>"bot"}, {:app=>"Googlebot", :match=>{"regex"=>"Googlebot/|Googlebot-Video/|Googlebot-Image/|^Feedfetcher-Google|google-xrawler|^Googlebot-News|^Google-Safety$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; http://www.google.com/bot.html)"}, {"userAgent"=>"Googlebot-Image/1.0"}, {"userAgent"=>"Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=4653447469100830145)"}, {"userAgent"=>"google-xrawler"}, {"userAgent"=>"Googlebot-News Audio"}, {"userAgent"=>"Google-Safety"}]}}, {:app=>"Gumball", :match=>{"regex"=>"^Gumball"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Gumball.fm Analytics Prefix Checker"}]}}, {:app=>"Headliner", :match=>{"regex"=>"^Headliner/.*\\+https://headliner\\.app$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Headliner/1.0.0 +https://headliner.app"}]}}, {:app=>"HubSpot Crawler", :match=>{"regex"=>"HubSpot Crawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; HubSpot Crawler; +https://www.hubspot.com)"}]}}, {:app=>"Internet Archive", :match=>{"regex"=>"Archive-It;|web\\.archive\\.org"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; special_archiver; Archive-It; http://archive-it.org/files/site-owners-special.html)"}, {"userAgent"=>"ia_archiver-web.archive.org"}]}}, {:app=>"iOS share menu", :match=>{"regex"=>"^SharingUIService/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SharingUIService/1 CFNetwork/1467.1 Darwin/23.0.0"}, {"userAgent"=>"SharingUIService/1 CFNetwork/1490.0.4 Darwin/23.2.0"}]}}, {:app=>"IPFS Podcasting Node", :match=>{"regex"=>"^IPFS Podcasting Node/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"IPFS Podcasting Node/0.6n (+https://ipfspodcasting.net)"}, {"userAgent"=>"IPFS Podcasting Node/0.7p,u,s (+https://ipfspodcasting.net)"}]}}, {:app=>"Jaunt", :match=>{"regex"=>"^Jaunt/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Jaunt/1.5"}]}}, {:app=>"l'Institut national de l'audiovisuel", :match=>{"regex"=>"INA dlweb"}, :platform=>"bot"}, {:app=>"Library and Archives Canada", :match=>{"regex"=>"LAC_IAHarvester/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; LAC_IAHarvester/3.3.0; +https://library-archives.canada.ca/eng/services/government-canada/web-social-media-preservation-program/Pages/web-archive.aspx)"}]}}, {:app=>"Libsyn", :match=>{"regex"=>"^Libsyn4"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Libsyn4-Download"}]}}, {:app=>"libwww-perl", :match=>{"regex"=>"^libwww-perl| libwww-perl"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"LWP::Simple/6.34 libwww-perl/6.34"}]}}, {:app=>"Livelap Crawler", :match=>{"regex"=>"LivelapBot"}, :platform=>"bot"}, {:app=>"ListenNotes", :match=>{"regex"=>"^ListenNotes/3\\.0 \\("}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"ListenNotes/3.0 (id=c1ef144e30174bf6ba1fedf9174ed908; +https://www.listennotes.com/about/)"}]}}, {:app=>"LTX71", :match=>{"regex"=>"^ltx71 "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"ltx71 - (http://ltx71.com/)"}]}}, {:app=>"MacOS share menu", :match=>{"regex"=>"^ShareSheetUI/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"ShareSheetUI/1 CFNetwork/1399.4 Darwin/22.1.0"}, {"userAgent"=>"ShareSheetUI/1 CFNetwork/1474 Darwin/23.0.0"}]}}, {:app=>"MauiBot", :match=>{"regex"=>"^MauiBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"MauiBot (crawler.feedback dc@gmail.com)"}]}}, {:app=>"Mastodon Bot", :match=>{"regex"=>"rb/.*Mastodon/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"http.rb/5.0.4 (Mastodon/3.5.3; +https://mastodon.xyz/) Bot"}]}}, {:app=>"Metacast Crawler", :match=>{"regex"=>"^Metacast/.* Crawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Metacast/1.0 Crawler +https://metacast.app"}]}}, {:app=>"Microsoft Bingbot", :match=>{"regex"=>"(BingPreview/|adidxbot/|[bB]ingbot/)"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534 (KHTML, like Gecko) BingPreview/1.0b"}, {"userAgent"=>"Mozilla/5.0 (compatible; Bingbot/2.0; +http://www.bing.com/bingbot.htm)"}]}}, {:app=>"Microsoft Office", :match=>{"regex"=>"ms-office; MSOffice"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/4.0 (compatible; ms-office; MSOffice 16)"}]}}, {:app=>"Mixcloud Podcast Importer", :match=>{"regex"=>"^MixcloudPodcastImporter/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"MixcloudPodcastImporter/2.0"}]}}, {:app=>"MJ12bot", :match=>{"regex"=>".*MJ12bot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)"}]}}, {:app=>"Mozilla Bot", :match=>{"regex"=>"^'?Mozilla(/5\\.0(\\.\\.\\.)?)?$|^\\(Mozilla/5\\.0\\)$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0"}, {"userAgent"=>"Mozilla"}, {"userAgent"=>"Mozilla/5.0..."}, {"userAgent"=>"'Mozilla/5.0"}, {"userAgent"=>"(Mozilla/5.0)"}]}}, {:app=>"MSN Bot", :match=>{"regex"=>"^msnbot/"}, :platform=>"bot"}, {:app=>"Neevabot", :match=>{"regex"=>".*Neevabot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Neevabot/1.0; https://neeva.com/neevabot)"}]}}, {:app=>"Netcraft Survey Agent", :match=>{"regex"=>" NetcraftSurveyAgent/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)"}]}}, {:app=>"Newspaper", :match=>{"regex"=>"^newspaper/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"newspaper/0.3.0"}]}}, {:app=>"OgScrper", :match=>{"regex"=>"OgScrper"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"OgScrper"}]}}, {:app=>"OkDownload", :match=>{"regex"=>"^OkDownload/"}, :platform=>"bot"}, {:app=>"OP3 Fetcher", :match=>{"regex"=>"^op3-fetcher/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"op3-fetcher/1.0 (https://op3.dev)"}]}}, {:app=>"OpenAI GPTBot", :match=>{"regex"=>"GPTBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)"}, {"userAgent"=>"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)"}]}}, {:app=>"Overcast feed parser", :match=>{"regex"=>"^Overcast/1\\.0 Podcast Sync"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Overcast/1.0 Podcast Sync"}, {"userAgent"=>"Overcast/1.0 Podcast Sync (+http://overcast.fm/)"}, {"userAgent"=>"Overcast/1.0 Podcast Sync (1 subscribers; feed-id=1234567; +http://overcast.fm/)"}]}}, {:app=>"Pandora RSS crawler", :match=>{"regex"=>"^PandoraRSSCrawler"}, :platform=>"bot"}, {:app=>"PaperLi", :match=>{"regex"=>"PaperLiBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)"}]}}, {:app=>"PetalBot", :match=>{"regex"=>"PetalBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot; https://aspiegel.com/petalbot)"}]}}, {:app=>"Pingdom", :match=>{"regex"=>"^Pingdom"}, :platform=>"bot"}, {:app=>"PlayerFM Podcast Sync", :match=>{"regex"=>"PlayerFM/.* Podcast Sync"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PlayerFM/1.0 Podcast Sync (0 subscribers; url=https://player.fm/series/series-2567872)"}]}}, {:app=>"Podbean Importer", :match=>{"regex"=>"^Podbean Importer"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podbean Importer"}]}}, {:app=>"Podcastindex.org", :match=>{"regex"=>"^Podcastindex\\.org/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcastindex.org/v0.3.3 (Aggrivate)"}]}}, {:app=>"Podcast de facto Standard", :match=>{"regex"=>"^PodcastStandard/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodcastStandard/0.1.0 (+https://podcast-standard.org)"}]}}, {:app=>"Podcast Archiver", :match=>{"regex"=>"^Podcast%20Archiver/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcast%20Archiver/3000000003 CFNetwork/1406.0.4 Darwin/22.4.0"}]}}, {:app=>"PodCloud", :match=>{"regex"=>"podCloud"}, :platform=>"bot"}, {:app=>"Podcorn", :match=>{"regex"=>"Podcorn/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podcorn/1.0"}]}}, {:app=>"PodderBot", :match=>{"regex"=>"PodderBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodderBot/1.0"}]}}, {:app=>"Podfollow", :match=>{"regex"=>"podfollowbot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 https://podfollow.com/crawling podfollowbot/1.0"}]}}, {:app=>"Podgrab", :match=>{"regex"=>"^Podgrab$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podgrab"}]}}, {:app=>"Podhound", :match=>{"regex"=>"PodhoundBeta"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodhoundBeta"}]}}, {:app=>"Podio Bot", :match=>{"regex"=>"^Podio/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podio/1.0"}]}}, {:app=>"Podnews", :match=>{"regex"=>"PodnewsBot"}, :platform=>"bot"}, {:app=>"Podnods Bot", :match=>{"regex"=>"(podnods-crawler|podnods)"}, :platform=>"bot"}, {:app=>"Podplay Podcast Sync", :match=>{"regex"=>"^Podplay-Podcast-Sync/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podplay-Podcast-Sync/1.0 (+https://podplay.com/articles/contact)"}]}}, {:app=>"Podroll Analyzer", :match=>{"regex"=>"^pdrl\\.fm "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"pdrl.fm Analyzer / 1.0.0"}]}}, {:app=>"Podscan Bot", :match=>{"regex"=>"^PodscanBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodscanBot/1.1.0 Ubuntu/22.04 (Linux) +https://podscan.fm"}, {"userAgent"=>"PodscanBot/1.1.0 Ubuntu/22.04 (Linux) Podscan.fm"}]}}, {:app=>"Podscribe", :match=>{"regex"=>"(^Adswizz-podscribe/|^Podscribe/)"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Adswizz-podscribe/1.0"}, {"userAgent"=>"Podscribe/1.1"}]}}, {:app=>"PodUptime", :match=>{"regex"=>"(PodUptime|PodUptimeBot)/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; PodUptime/1.0; +https://poduptime.com)"}, {"userAgent"=>"Mozilla/5.0 (compatible; PodUptimeBot/1.0; +https://poduptime.com; rid:5c02fc9b-2660-4d3e-a352-de1a1aabdc0a)"}]}}, {:app=>"Podverse Feed Parser", :match=>{"regex"=>"^Podverse/Feed Parser"}, :platform=>"bot"}, {:app=>"PodvineBot", :match=>{"regex"=>"^PodvineBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PodvineBot/4.0.1 (www.podvine.com)"}]}}, {:app=>"PostRank Bot", :match=>{"regex"=>"^PostRank/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"PostRank/2.0 (postrank.com; 1 subscribers)"}]}}, {:app=>"PodwatchPro", :match=>{"regex"=>"Podwatch-Pro Crawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Podwatch-Pro Crawler"}]}}, {:app=>"Puppeteer", :match=>{"regex"=>" HeadlessChrome/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/105.0.5173.0 Safari/537.36"}]}}, {:app=>"python-requests", :match=>{"regex"=>"python-requests"}, :platform=>"bot"}, {:app=>"Qiniu spider", :match=>{"regex"=>"qiniu.*spider"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"z0-qiniu-imgstg-spider-1.0"}]}}, {:app=>"Radioline Stream Monitoring", :match=>{"regex"=>"^Radioline Stream Monitoring"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Radioline Stream Monitoring (tech@radioline.co)"}]}}, {:app=>"RedCircle", :match=>{"regex"=>"RedCircle"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"RedCircle"}]}}, {:app=>"Rephonic", :match=>{"regex"=>"^Rephonic/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Rephonic/1.0"}]}}, {:app=>"Ridder Bot", :match=>{"regex"=>" RidderBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co)"}]}}, {:app=>"RSS to Telegram", :match=>{"regex"=>"^RSStT"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"RSStT/2.2.1 RSS Reader"}]}}, {:app=>"RSSRadio", :match=>{"regex"=>"^RSSRadio \\("}, :platform=>"bot"}, {:app=>"Ruby Mechanize", :match=>{"regex"=>"^Mechanize|[ -]Mechanize/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Mechanize/2.7.4)"}, {"userAgent"=>"WWW-Mechanize/1.72"}]}}, {:app=>"Screaming Frog SEO Spider", :match=>{"regex"=>"^Screaming Frog "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Screaming Frog SEO Spider/5.1"}]}}, {:app=>"SearchAtlas.com SEO Crawler", :match=>{"regex"=>"^SearchAtlas.*Crawler"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SearchAtlas.com SEO Crawler"}]}}, {:app=>"SEMrushBot", :match=>{"regex"=>"SemrushBot/|^SEMrushBot$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; SemrushBot/6~bl; http://www.semrush.com/bot.html)"}, {"userAgent"=>"SEMrushBot"}]}}, {:app=>"Serendeputy", :match=>{"regex"=>"SerendeputyBot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)"}]}}, {:app=>"Simplecast", :match=>{"regex"=>"^Simplecast$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Simplecast"}]}}, {:app=>"Slack Bot", :match=>{"regex"=>"^Slackbot 1\\.0"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Slackbot 1.0 ( https://api.slack.com/robots)"}]}}, {:app=>"Snapchat Bot", :match=>{"regex"=>"://developers\\.snap\\.com/robots"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots"}]}}, {:app=>"SoundOn Bot", :match=>{"regex"=>"^SoundOn/[\\d.]+\\s+\\(bot"}, :platform=>"bot"}, {:app=>"Speechmatics ASR", :match=>{"regex"=>"^Speechmatics-API/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Speechmatics-API/2.0"}]}}, {:app=>"Spotify cache service", :match=>{"regex"=>"^Spotify/1\\.0$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Spotify/1.0"}]}}, {:app=>"Stitcher Bot", :match=>{"regex"=>"^StitcherBot"}, :platform=>"bot"}, {:app=>"Substack Content Fetcher", :match=>{"regex"=>"^SubstackContentFetch/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SubstackContentFetch/1.0 (https://substack.com/)"}]}}, {:app=>"SupportingCast", :match=>{"regex"=>"^SupportingCast(/.*)?$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"SupportingCast/1.0"}, {"userAgent"=>"SupportingCast"}]}}, {:app=>"Timpi search crawler", :match=>{"regex"=>"Timpibot/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Timpibot/0.8 ( http://www.timpi.io)"}]}}, {:app=>"Tiny Tiny RSS", :match=>{"regex"=>"^Tiny Tiny RSS/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Tiny Tiny RSS/22.12-c30b24d09 (Unsupported) (https://tt-rss.org/)"}]}}, {:app=>"Transistor", :match=>{"regex"=>"^Transistor\\.fm/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Transistor.fm/1.0"}]}}, {:app=>"Trendsmap Resolver", :match=>{"regex"=>"TrendsmapResolver/"}, :platform=>"bot"}, {:app=>"Triton Digital Podcast Sync", :match=>{"regex"=>"^Triton Digital Podcast Sync$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Triton Digital Podcast Sync"}]}}, {:app=>"Twingly Bot", :match=>{"regex"=>"Twingly Recon;"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; Twingly Recon; twingly.com)"}]}}, {:app=>"Twitterbot", :match=>{"regex"=>"^Twitterbot"}, :platform=>"bot"}, {:app=>"Typhoeus", :match=>{"regex"=>"^Typhoeus"}, :platform=>"bot"}, {:app=>"UCast", :match=>{"regex"=>"^UCast/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"UCast/1.0 Podcast Sync (1 subscribers; feed-id=aHR0cHM6Ly9mZWVkcy5idXp6c3Byb3V0LmNvbS8yMDg0OTQucnNzEAEBAD32-0242-42AC-8583-21E9BAD8C544; +http://www.ucastapp.com/)"}]}}, {:app=>"UptimeRobot", :match=>{"regex"=>"UptimeRobot/\\d"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)"}]}}, {:app=>"TelegramBot", :match=>{"regex"=>"^TelegramBot "}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"TelegramBot (like TwitterBot)"}]}}, {:app=>"theTradeDesk Content Web Scraper", :match=>{"regex"=>"TTD-Content"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; TTD-Content; +https://www.thetradedesk.com/general/ttd-content)"}]}}, {:app=>"Veritone Engine Toolkit", :match=>{"regex"=>"^veritone/engine-toolkit"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"veritone/engine-toolkit-3.0"}]}}, {:app=>"Vurbl", :match=>{"regex"=>"VurblBot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 https://vurbl.com VurblBot/1.0"}]}}, {:app=>"Wget", :match=>{"regex"=>"Wget"}, :platform=>"bot"}, {:app=>"weborama", :match=>{"regex"=>"^weborama-fetcher"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"weborama-fetcher (+http://www.weborama.com)"}]}}, {:app=>"Windows Crawler", :match=>{"regex"=>"^ZDM/.*Windows"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"ZDM/4.0; Windows Mobile 7.0;"}]}}, {:app=>"Windows PowerShell", :match=>{"regex"=>"WindowsPowerShell/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (Windows NT; Windows NT 10.0; en-US) WindowsPowerShell/5.1.19041.2673"}]}}, {:app=>"WordPress", :match=>{"regex"=>"^WordPress"}, :platform=>"bot"}, {:app=>"wsrv.nl", :match=>{"regex"=>"https?://wsrv.nl/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Mozilla/5.0 (compatible; ImageFetcher/9.0; +http://wsrv.nl/)"}]}}, {:app=>"YaCy", :match=>{"regex"=>"^yacybot"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"yacybot (/global; amd64 Linux 5.9.8-zen1-1-zen; java 1.8.0_265; Europe/de) http://yacy.net/bot.html"}]}}, {:app=>"Yahoo Crawler", :match=>{"regex"=>"Yahoo! Slurp"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Crawler Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"}]}}, {:app=>"YandexBot", :match=>{"regex"=>"YandexBot/"}, :platform=>"bot"}, {:app=>"Zapier", :match=>{"regex"=>"^Zapier$"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Zapier"}]}}, {:app=>"Zencast", :match=>{"regex"=>"^Zencastr/"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Zencastr/2.0"}]}}].freeze \ No newline at end of file diff --git a/spec/detection_rules_custom_bots.rb b/spec/detection_rules_custom_bots.rb index 973f486..0d0a193 100644 --- a/spec/detection_rules_custom_bots.rb +++ b/spec/detection_rules_custom_bots.rb @@ -1,3 +1,3 @@ # DO NOT EDIT THIS FILE - it gets automatically generated by running "bin/parse-rules" -CUSTOM_BOTS_RULES = [{:app=>"curl", :match=>{"regex"=>"^Faraday"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Faraday v1.0.1"}]}}, {:app=>"curl", :match=>{"regex"=>"like curl"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Faraday for Podigee (like curl)"}]}}].freeze \ No newline at end of file +CUSTOM_BOTS_RULES = [{:app=>"Apache HTTPClient", :match=>{"regex"=>"^Apache\\-HttpClient"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Apache-HttpClient/5.1.4 (Java/1.8.0_144)"}, {"userAgent"=>"Apache-HttpClient/5.1.4 (Java/1.8.0_91)"}]}}, {:app=>"curl", :match=>{"regex"=>"^Faraday"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Faraday v1.0.1"}]}}, {:app=>"curl", :match=>{"regex"=>"like curl"}, :platform=>"bot", :test=>{"userAgents"=>[{"userAgent"=>"Faraday for Podigee (like curl)"}]}}].freeze \ No newline at end of file