idaholab#602, include support for sending Zeek logs to Kafka

Dockerfiles/zeek.Dockerfile

@@ -87,6 +87,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
       libmaxminddb0 \
       libpcap-dev \
       libpcap0.8 \
+      librdkafka-dev \
       libssl-dev \
       libssl3 \
       libtcmalloc-minimal4 \
@@ -174,7 +175,7 @@ RUN groupadd --gid ${DEFAULT_GID} ${PUSER} && \
 
 # sanity checks to make sure the plugins installed and copied over correctly
 # these ENVs should match the third party scripts/plugins installed by zeek_install_plugins.sh
-ENV ZEEK_THIRD_PARTY_PLUGINS_GREP  "(Zeek::Spicy|ANALYZER_SPICY_OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SPICY_HART_IP_UDP|ANALYZER_SPICY_HART_IP_TCP|ANALYZER_SYNCHROPHASOR_TCP|ANALYZER_GENISYS_TCP|ANALYZER_SPICY_GE_SRTP|ANALYZER_SPICY_PROFINET_IO_CM|ANALYZER_S7COMM_TCP|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)"
+ENV ZEEK_THIRD_PARTY_PLUGINS_GREP  "(Zeek::Spicy|ANALYZER_SPICY_OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SPICY_HART_IP_UDP|ANALYZER_SPICY_HART_IP_TCP|ANALYZER_SYNCHROPHASOR_TCP|ANALYZER_GENISYS_TCP|ANALYZER_SPICY_GE_SRTP|ANALYZER_SPICY_PROFINET_IO_CM|ANALYZER_S7COMM_TCP|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS|Seiso::Kafka)"
 ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP  "(bro-is-darknet/main|bro-simple-scan/scan|bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja4/main|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)"
 
 RUN mkdir -p /tmp/logs && \
@@ -183,7 +184,7 @@ RUN mkdir -p /tmp/logs && \
     export ZEEK_THIRD_PARTY_SCRIPTS_COUNT=$(echo "$ZEEK_THIRD_PARTY_SCRIPTS_GREP" | grep -P -o "\([^)]+\)" | head -n 1 | sed "s/^(//" | sed "s/)$//" | tr '|' '\n' | wc -l) && \
     "$ZEEK_DIR"/bin/zeek-offline -NN local >zeeknn.log 2>/dev/null && \
       bash -c "(( $(grep -cP "$ZEEK_THIRD_PARTY_PLUGINS_GREP" zeeknn.log) >= $ZEEK_THIRD_PARTY_PLUGINS_COUNT)) && echo $ZEEK_THIRD_PARTY_PLUGINS_COUNT' Zeek plugins loaded correctly' || (echo 'One or more Zeek plugins did not load correctly' && cat zeeknn.log && exit 1)" && \
-    "$ZEEK_DIR"/bin/zeek-offline -C -r /tmp/pcaps/udp.pcap local policy/misc/loaded-scripts 2>/dev/null && \
+    "$ZEEK_DIR"/bin/zeek-offline -C -r /tmp/pcaps/udp.pcap local policy/misc/loaded-scripts >loaded_scripts.log 2>/dev/null && \
       bash -c "(( $(grep -cP "$ZEEK_THIRD_PARTY_SCRIPTS_GREP" loaded_scripts.log) == $ZEEK_THIRD_PARTY_SCRIPTS_COUNT)) && echo $ZEEK_THIRD_PARTY_SCRIPTS_COUNT' Zeek scripts loaded correctly' || (echo 'One or more Zeek scripts did not load correctly' && cat loaded_scripts.log && exit 1)" && \
     cd /tmp && \
     rm -rf /tmp/logs /tmp/pcaps
@@ -249,8 +250,6 @@ ARG ZEEK_DISABLE_SSL_VALIDATE_CERTS=
 ARG ZEEK_DISABLE_TRACK_ALL_ASSETS=
 ARG ZEEK_DISABLE_DETECT_ROUTERS=true
 ARG ZEEK_DISABLE_BEST_GUESS_ICS=true
-# TODO: assess spicy-analyzer that replace built-in Zeek parsers
-# for now, disable them by default when a Zeek parser exists
 ARG ZEEK_DISABLE_SPICY_IPSEC=
 ARG ZEEK_DISABLE_SPICY_LDAP=
 ARG ZEEK_DISABLE_SPICY_OPENVPN=
@@ -260,6 +259,9 @@ ARG ZEEK_DISABLE_SPICY_TAILSCALE=
 ARG ZEEK_DISABLE_SPICY_TFTP=
 ARG ZEEK_DISABLE_SPICY_WIREGUARD=
 ARG ZEEK_SYNCHROPHASOR_DETAILED=
+ARG ZEEK_KAFKA_ENABLED=
+ARG ZEEK_KAFKA_BROKERS=kafka.local:9091
+ARG ZEEK_KAFKA_TOPIC=zeek
 
 ENV ZEEK_DISABLE_STATS $ZEEK_DISABLE_STATS
 ENV ZEEK_DISABLE_HASH_ALL_FILES $ZEEK_DISABLE_HASH_ALL_FILES
@@ -278,6 +280,9 @@ ENV ZEEK_DISABLE_SPICY_TAILSCALE $ZEEK_DISABLE_SPICY_TAILSCALE
 ENV ZEEK_DISABLE_SPICY_TFTP $ZEEK_DISABLE_SPICY_TFTP
 ENV ZEEK_DISABLE_SPICY_WIREGUARD $ZEEK_DISABLE_SPICY_WIREGUARD
 ENV ZEEK_SYNCHROPHASOR_DETAILED $ZEEK_SYNCHROPHASOR_DETAILED
+ENV ZEEK_KAFKA_ENABLED $ZEEK_KAFKA_ENABLED
+ENV ZEEK_KAFKA_BROKERS $ZEEK_KAFKA_BROKERS
+ENV ZEEK_KAFKA_TOPIC $ZEEK_KAFKA_TOPIC
 
 # This is in part to handle an issue when running with rootless podman and
 #   "userns_mode: keep-id". It seems that anything defined as a VOLUME

config/zeek.env.example

@@ -99,3 +99,6 @@ ZEEK_SYNCHROPHASOR_DETAILED=
 ZEEK_GENISYS_PORTS=
 ZEEK_ENIP_PORTS=
 ZEEK_DISABLE_BEST_GUESS_ICS=true
+ZEEK_KAFKA_ENABLED=
+ZEEK_KAFKA_BROKERS=kafka.local:9091
+ZEEK_KAFKA_TOPIC=zeek

hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot

@@ -9,7 +9,7 @@ apt-get -y --purge remove \
   libc6-dbg \
   ninja-build \
   sparse \
-  $(dpkg --get-selections | grep -Pv "(^(dpkg|libbroker|libc6|libcrypt|libdbus|libffi|libfl|libgoogle-perftools|libgcc|libkrb5|libmaxminddb|libncurses|libnsl|libobjc|libomp|libpcap|libssl|libstdc|libtinfo|libtirpc|libunwind|libxml|libyaml|libz|linux-libc|python3|zeek|zlib1g)|deinstall$)" | cut -f1 | grep -P -- '-dev(:\w+)?$') || true
+  $(dpkg --get-selections | grep -Pv "(^(dpkg|libbroker|libc6|libcrypt|libdbus|libffi|libfl|libgoogle-perftools|libgcc|libkrb5|librdkafka|libmaxminddb|libncurses|libnsl|libobjc|libomp|libpcap|libssl|libstdc|libtinfo|libtirpc|libunwind|libxml|libyaml|libz|linux-libc|python3|zeek|zlib1g)|deinstall$)" | cut -f1 | grep -P -- '-dev(:\w+)?$') || true
 rm -rf /var/spool/ccache
 
 # remove unwanted packages

hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek

@@ -44,6 +44,10 @@ global disable_ics_profinet_io_cm = (getenv("ZEEK_DISABLE_ICS_PROFINET_IO_CM") =
 global disable_ics_s7comm = (getenv("ZEEK_DISABLE_ICS_S7COMM") == true_regex) ? T : F;
 global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == true_regex) ? T : F;
 
+global zeek_kafka_enabled = (getenv("ZEEK_KAFKA_ENABLED") == true_regex) ? T : F;
+global zeek_kafka_brokers = getenv("ZEEK_KAFKA_BROKERS");
+global zeek_kafka_topic = getenv("ZEEK_KAFKA_TOPIC");
+
 redef Broker::default_listen_address = "127.0.0.1";
 redef ignore_checksums = T;
 
@@ -344,3 +348,13 @@ hook PacketAnalyzer::ECAT::log_policy_ecat_arp(
   filter: Log::Filter) {
   break;
 }
+
+@if (zeek_kafka_enabled)
+ @load packages/zeek-kafka
+ redef Kafka::send_all_active_logs = T;
+ redef Kafka::topic_name = zeek_kafka_topic;
+ redef Kafka::tag_json = T;
+ redef Kafka::kafka_conf = table(
+     ["metadata.broker.list"] = zeek_kafka_brokers
+);
+@endif

hedgehog-iso/config/package-lists/build.list.chroot

@@ -12,6 +12,7 @@ libkrb5-dev
 libmagic-dev
 libmaxminddb-dev
 libnl-3-dev
+librdkafka-dev
 libpcap-dev
 libsodium-dev
 libssl-dev

hedgehog-iso/interface/sensor_ctl/control_vars.conf

@@ -151,6 +151,9 @@ export ZEEK_SYNCHROPHASOR_DETAILED=
 export ZEEK_GENISYS_PORTS=
 export ZEEK_ENIP_PORTS=
 export ZEEK_DISABLE_BEST_GUESS_ICS=true
+export ZEEK_KAFKA_ENABLED=
+export ZEEK_KAFKA_BROKERS=kafka.local:9091
+export ZEEK_KAFKA_TOPIC=zeek
 
 # Suricata
 export SURICATA_CUSTOM_RULES_ONLY=false

shared/bin/suricata_config_populate.py

@@ -1282,7 +1282,7 @@ def main():
                     tmpLogDir,
                     '-T',
                 ],
-                debug=args.verbose > logging.DEBUG,
+                debug=args.verbose <= logging.DEBUG,
                 logger=logging,
             )
             logging.info(f'suricata configuration test returned {script_return_code}')

shared/bin/zeek_install_plugins.sh

@@ -63,10 +63,15 @@ function zkg_install_github_repo() {
         export "$ENV"
       done
     fi
+    EXTRA_ZKG_PARAMS=()
+    if [[ "$REPO_URL" =~ "zeek-kafka" ]]; then
+      EXTRA_ZKG_PARAMS+=( --user-var )
+      EXTRA_ZKG_PARAMS+=( LIBRDKAFKA_ROOT=/usr/include/librdkafka )
+    fi
     if [[ -n $REPO_LATEST_RELEASE ]]; then
-      zkg install --nodeps --force --skiptests --version "$REPO_LATEST_RELEASE" "$REPO_URL"
+      zkg install --nodeps --force --skiptests "${EXTRA_ZKG_PARAMS[@]}" --version "$REPO_LATEST_RELEASE" "$REPO_URL"
     else
-      zkg install --nodeps --force --skiptests "$REPO_URL"
+      zkg install --nodeps --force --skiptests "${EXTRA_ZKG_PARAMS[@]}" "$REPO_URL"
     fi
   fi
 }
@@ -127,6 +132,7 @@ ZKG_GITHUB_URLS=(
   "https://github.com/ncsa/bro-simple-scan"
   "https://github.com/precurse/zeek-httpattacks"
   "https://github.com/mmguero-dev/GQUIC_Protocol_Analyzer"
+  "https://github.com/SeisoLLC/zeek-kafka"
   "https://github.com/zeek/spicy-tftp"
   "https://github.com/zeek/spicy-zip"
 )

zeek/config/local.zeek

@@ -44,6 +44,10 @@ global disable_ics_profinet_io_cm = (getenv("ZEEK_DISABLE_ICS_PROFINET_IO_CM") =
 global disable_ics_s7comm = (getenv("ZEEK_DISABLE_ICS_S7COMM") == true_regex) ? T : F;
 global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == true_regex) ? T : F;
 
+global zeek_kafka_enabled = (getenv("ZEEK_KAFKA_ENABLED") == true_regex) ? T : F;
+global zeek_kafka_brokers = getenv("ZEEK_KAFKA_BROKERS");
+global zeek_kafka_topic = getenv("ZEEK_KAFKA_TOPIC");
+
 redef Broker::default_listen_address = "127.0.0.1";
 redef ignore_checksums = T;
 
@@ -344,3 +348,13 @@ hook PacketAnalyzer::ECAT::log_policy_ecat_arp(
   filter: Log::Filter) {
   break;
 }
+
+@if (zeek_kafka_enabled)
+ @load packages/zeek-kafka
+ redef Kafka::send_all_active_logs = T;
+ redef Kafka::topic_name = zeek_kafka_topic;
+ redef Kafka::tag_json = T;
+ redef Kafka::kafka_conf = table(
+     ["metadata.broker.list"] = zeek_kafka_brokers
+);
+@endif