From 4b1a1b5d2b6b39573ead3c30589fceeaae2247e7 Mon Sep 17 00:00:00 2001 From: Manabu Date: Sat, 7 Sep 2019 01:08:15 +0900 Subject: [PATCH] add fluent example --- .gitignore | 1 + Dockerfile | 2 +- README.md | 2 +- entrypoint.sh | 10 ++++---- example/fluent/docker-compose.yml | 38 +++++++++++++++++++++++++++++++ example/fluent/run.sh | 16 +++++++++++++ example/kafka/TBD | 0 example/kafka/docker-compose.yml | 33 +++++++++++++++++++++++++++ 8 files changed, 95 insertions(+), 7 deletions(-) mode change 100644 => 100755 entrypoint.sh create mode 100644 example/fluent/docker-compose.yml create mode 100755 example/fluent/run.sh create mode 100644 example/kafka/TBD create mode 100644 example/kafka/docker-compose.yml diff --git a/.gitignore b/.gitignore index 722d5e7..3f085c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .vscode +example/*/volumes \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f7256da..48aa4eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ ENV DTAP_IPV4_MASK 24 ENV DTAP_IPV6_MASK 48 ENV DTAP_ENABLE_ECS "false" ENV DTAP_ENABLE_HASH_IP "false" -ENV DTAP_ENABLE_HASH_SALT "" +ENV DTAP_HASH_SALT "" COPY entrypoint.sh / COPY --from=builder /build/cmd/dtap/dtap /usr/bin/dtap diff --git a/README.md b/README.md index 59d0adb..a33f13e 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ Make flatting DNSTAP message,And it forawrd to nats host. ``` [[OutputNats]] -Host = "nats://kafka.example.jp:5000"] +Host = "nats://kafka.example.jp:5000" Subject = "dnstap" User = "dnstap" Password = "hogehoge diff --git a/entrypoint.sh b/entrypoint.sh old mode 100644 new mode 100755 index 1809e13..afb5413 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -28,18 +28,18 @@ EOS Path="${DTAP_OUTPUT_UNIX_SOCKET}" EOS fi - if [ "${DTAP_OUTPUT_FLUENT_HOST}"] ; then + if [ "${DTAP_OUTPUT_FLUENT_HOST}" != "" ] ; then cat <<- EOS >> /etc/dtap/dtap.conf [[OutputFluent]] Host = "${DTAP_OUTPUT_FLUENT_HOST}" Port = ${DTAP_OUTPUT_FLUENT_PORT} Tag = "${DTAP_OUTPUT_FLUENT_TAG}" - [OutputKafka.flat] + [OutputFluent.flat] IPv4Mask = ${DTAP_IPV4_MASK} IPv6Mask = ${DTAP_IPV6_MASK} EnableECS = ${DTAP_ENABLE_ECS} EnableHashIP = ${DTAP_ENABLE_HASH_IP} - IPHashSaltPath = "${DTAP_ENABLE_HASH_SALT}" + IPHashSaltPath = "${DTAP_HASH_SALT}" EOS fi @@ -54,7 +54,7 @@ EOS IPv6Mask = ${DTAP_IPV6_MASK} EnableECS = ${DTAP_ENABLE_ECS} EnableHashIP = ${DTAP_ENABLE_HASH_IP} - IPHashSaltPath = "${DTAP_ENABLE_HASH_SALT}" + IPHashSaltPath = "${DTAP_HASH_SALT}" EOS fi if [ "${DTAP_OUTPUT_NATS_HOST}" != "" ] ; then @@ -70,7 +70,7 @@ EOS IPv6Mask = ${DTAP_IPV6_MASK} EnableECS = ${DTAP_ENABLE_ECS} EnableHashIP = ${DTAP_ENABLE_HASH_IP} - IPHashSaltPath = "${DTAP_ENABLE_HASH_SALT}" + IPHashSaltPath = "${DTAP_HASH_SALT}" EOS fi fi diff --git a/example/fluent/docker-compose.yml b/example/fluent/docker-compose.yml new file mode 100644 index 0000000..0397715 --- /dev/null +++ b/example/fluent/docker-compose.yml @@ -0,0 +1,38 @@ +version: "3" +services: + fluent-bit: + image: fluent/fluent-bit:latest + ports: + - "24224:24224/tcp" + command: /fluent-bit/bin/fluent-bit -v -i forward -o es -p Host=elasticsearch -p Logstash_Format=On -p Logstash_Prefix=dnstap -p Type=dnstap -p Generate_ID=On -m '*' + elasticsearch: + image: elasticsearch:7.3.1 + ports: + - "9200:9200/tcp" + environment: + discovery.type: single-node + ES_JAVA_OPTS: -Xms512m -Xmx512m + kibana: + image: kibana:7.3.1 + ports: + - "5601:5601/tcp" + depends_on: + - elasticsearch + unbound: + image: mimuret/unbound:latest + ports: + - "53:53/tcp" + - "53:53/udp" + volumes: + - "./volumes/run:/unbound/var/run" + dtap: + image: mimuret/dtap:latest + volumes: + - "./volumes/run:/unbound/var/run" + command: /usr/bin/dtap -c /etc/dtap/dtap.conf -d debug + environment: + DTAP_INPUT_UNIX_SOCKET: /unbound/var/run/dnstap.sock + DTAP_OUTPUT_FLUENT_HOST: fluent-bit + DTAP_OUTPUT_FLUENT_TAG: query + depends_on: + - fluent-bit diff --git a/example/fluent/run.sh b/example/fluent/run.sh new file mode 100755 index 0000000..e01f49a --- /dev/null +++ b/example/fluent/run.sh @@ -0,0 +1,16 @@ +#!/bin/sh + + +docker-compose up -d + +curl http://localhost:9200 > /dev/null 2>&1 +while [ $? -ne 0 ] +do + curl http://localhost:9200 > /dev/null 2>&1 + sleep 5 +done + +curl http://localhost:9200/_template/dtap -H "Content-Type: application/json" -XPUT -d '@../../misc/template.json' -v + +dig @localhost github.com + diff --git a/example/kafka/TBD b/example/kafka/TBD new file mode 100644 index 0000000..e69de29 diff --git a/example/kafka/docker-compose.yml b/example/kafka/docker-compose.yml new file mode 100644 index 0000000..6213016 --- /dev/null +++ b/example/kafka/docker-compose.yml @@ -0,0 +1,33 @@ +version: "3" +services: + kafka: + image: spotify/kafka + ports: + - "2181:2181" + - "9092:9092" + KAFKA_ADVERTISED_HOST_NAME: kafka + elasticsearch: + image: elasticsearch:7.3.1 + ports: + - "9200:9200/tcp" + environment: + discovery.type: single-node + ES_JAVA_OPTS: -Xms512m -Xmx512m + unbound: + image: mimuret/unbound:latest + ports: + - "53:53/tcp" + - "53:53/udp" + volumes: + - "./volumes/run:/unbound/var/run" + dtap: + image: mimuret/dtap:latest + volumes: + - "./volumes/run:/unbound/var/run" + command: /usr/bin/dtap -c /etc/dtap/dtap.conf -d debug + environment: + DTAP_INPUT_UNIX_SOCKET: /unbound/var/run/dnstap.sock + DTAP_OUTPUT_KAFKA_HOSTS: kafka + DTAP_OUTPUT_KAFKA_TOPIC: query + depends_on: + - kafka