add fluent example

.gitignore

@@ -1 +1,2 @@
 .vscode
+example/*/volumes

Dockerfile

@@ -27,7 +27,7 @@ ENV DTAP_IPV4_MASK 24
 ENV DTAP_IPV6_MASK 48
 ENV DTAP_ENABLE_ECS "false"
 ENV DTAP_ENABLE_HASH_IP "false"
-ENV DTAP_ENABLE_HASH_SALT ""
+ENV DTAP_HASH_SALT ""
 
 COPY entrypoint.sh /
 COPY --from=builder /build/cmd/dtap/dtap /usr/bin/dtap

README.md

@@ -104,7 +104,7 @@ Make flatting DNSTAP message,And it forawrd to nats host.
 
 ```
 [[OutputNats]]
-Host = "nats://kafka.example.jp:5000"]
+Host = "nats://kafka.example.jp:5000"
 Subject  = "dnstap"
 User = "dnstap"
 Password = "hogehoge

entrypoint.sh

@@ -28,18 +28,18 @@ EOS
   Path="${DTAP_OUTPUT_UNIX_SOCKET}"
 EOS
   fi
-  if [ "${DTAP_OUTPUT_FLUENT_HOST}"] ; then
+  if [ "${DTAP_OUTPUT_FLUENT_HOST}" != "" ] ; then
     cat <<- EOS >> /etc/dtap/dtap.conf
 [[OutputFluent]]
   Host = "${DTAP_OUTPUT_FLUENT_HOST}"
   Port = ${DTAP_OUTPUT_FLUENT_PORT}
   Tag = "${DTAP_OUTPUT_FLUENT_TAG}"
-  [OutputKafka.flat]
+  [OutputFluent.flat]
     IPv4Mask = ${DTAP_IPV4_MASK}
     IPv6Mask = ${DTAP_IPV6_MASK}
     EnableECS = ${DTAP_ENABLE_ECS}
     EnableHashIP = ${DTAP_ENABLE_HASH_IP}
-    IPHashSaltPath = "${DTAP_ENABLE_HASH_SALT}"
+    IPHashSaltPath = "${DTAP_HASH_SALT}"
 EOS
 
   fi
@@ -54,7 +54,7 @@ EOS
     IPv6Mask = ${DTAP_IPV6_MASK}
     EnableECS = ${DTAP_ENABLE_ECS}
     EnableHashIP = ${DTAP_ENABLE_HASH_IP}
-    IPHashSaltPath = "${DTAP_ENABLE_HASH_SALT}"
+    IPHashSaltPath = "${DTAP_HASH_SALT}"
 EOS
   fi
   if [ "${DTAP_OUTPUT_NATS_HOST}" != "" ] ; then
@@ -70,7 +70,7 @@ EOS
     IPv6Mask = ${DTAP_IPV6_MASK}
     EnableECS = ${DTAP_ENABLE_ECS}
     EnableHashIP = ${DTAP_ENABLE_HASH_IP}
-    IPHashSaltPath = "${DTAP_ENABLE_HASH_SALT}"
+    IPHashSaltPath = "${DTAP_HASH_SALT}"
 EOS
   fi
 fi

example/fluent/docker-compose.yml

@@ -0,0 +1,38 @@
+version: "3"
+services:
+  fluent-bit:
+    image: fluent/fluent-bit:latest
+    ports:
+      - "24224:24224/tcp"
+    command: /fluent-bit/bin/fluent-bit -v -i forward -o es -p Host=elasticsearch -p Logstash_Format=On -p Logstash_Prefix=dnstap -p Type=dnstap -p Generate_ID=On -m '*'
+  elasticsearch:
+    image: elasticsearch:7.3.1
+    ports:
+      - "9200:9200/tcp"
+    environment:
+      discovery.type: single-node
+      ES_JAVA_OPTS: -Xms512m -Xmx512m
+  kibana:
+    image: kibana:7.3.1
+    ports:
+      - "5601:5601/tcp"
+    depends_on:
+      - elasticsearch
+  unbound:
+    image: mimuret/unbound:latest
+    ports:
+      - "53:53/tcp"
+      - "53:53/udp"
+    volumes:
+      - "./volumes/run:/unbound/var/run"
+  dtap:
+    image: mimuret/dtap:latest
+    volumes:
+      - "./volumes/run:/unbound/var/run"
+    command: /usr/bin/dtap -c /etc/dtap/dtap.conf -d debug
+    environment:
+      DTAP_INPUT_UNIX_SOCKET: /unbound/var/run/dnstap.sock
+      DTAP_OUTPUT_FLUENT_HOST: fluent-bit
+      DTAP_OUTPUT_FLUENT_TAG: query
+    depends_on:
+      - fluent-bit

example/fluent/run.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+
+docker-compose up -d 
+
+curl http://localhost:9200 > /dev/null 2>&1
+while [ $? -ne 0 ] 
+do
+  curl http://localhost:9200 > /dev/null 2>&1
+  sleep 5
+done
+
+curl http://localhost:9200/_template/dtap -H "Content-Type: application/json" -XPUT -d '@../../misc/template.json' -v
+
+dig @localhost github.com
+

example/kafka/docker-compose.yml

@@ -0,0 +1,33 @@
+version: "3"
+services:
+  kafka:
+    image: spotify/kafka
+    ports:
+      - "2181:2181"
+      - "9092:9092"
+    KAFKA_ADVERTISED_HOST_NAME: kafka
+  elasticsearch:
+    image: elasticsearch:7.3.1
+    ports:
+      - "9200:9200/tcp"
+    environment:
+      discovery.type: single-node
+      ES_JAVA_OPTS: -Xms512m -Xmx512m
+  unbound:
+    image: mimuret/unbound:latest
+    ports:
+      - "53:53/tcp"
+      - "53:53/udp"
+    volumes:
+      - "./volumes/run:/unbound/var/run"
+  dtap:
+    image: mimuret/dtap:latest
+    volumes:
+      - "./volumes/run:/unbound/var/run"
+    command: /usr/bin/dtap -c /etc/dtap/dtap.conf -d debug
+    environment:
+      DTAP_INPUT_UNIX_SOCKET: /unbound/var/run/dnstap.sock
+      DTAP_OUTPUT_KAFKA_HOSTS: kafka
+      DTAP_OUTPUT_KAFKA_TOPIC: query
+    depends_on:
+      - kafka