Merge pull request #1 from SmartColumbusOS/initial_implementation

First implementation of divo-kafka

.formatter.exs

@@ -0,0 +1,5 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"],
+  line_length: 120
+]

.gitignore

@@ -0,0 +1,24 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+divo_kafka-*.tar
+

Dockerfile

@@ -0,0 +1,11 @@
+FROM bitwalker/alpine-elixir:1.8.1
+ARG HEX_TOKEN
+COPY . /app
+WORKDIR /app
+RUN mix local.hex --force \
+    && mix local.rebar --force \
+    && mix hex.organization auth smartcolumbus_os --key ${HEX_TOKEN} \
+    && mix deps.get \
+    && mix test \
+    && mix format --check-formatted \
+    && mix credo

Jenkinsfile

@@ -0,0 +1,30 @@
+library(
+    identifier: 'pipeline-lib@4.3.4',
+    retriever: modernSCM([$class: 'GitSCMSource',
+                          remote: 'https://github.com/SmartColumbusOS/pipeline-lib',
+                          credentialsId: 'jenkins-github-user'])
+)
+
+def image
+
+node('infrastructure') {
+    ansiColor('xterm') {
+        scos.doCheckoutStage()
+
+        stage('Build') {
+            withCredentials([string(credentialsId: 'hex-read', variable: 'HEX_TOKEN')]) {
+                image = docker.build("divo-kafka:${env.GIT_COMMIT_HASH}", "--build-arg HEX_TOKEN=$HEX_TOKEN .")
+            }
+        }
+
+        stage('Test') {
+            image.run('--rm', 'mix test')
+        }
+
+        scos.doStageIf(scos.changeset.isRelease, "Publish") {
+            withCredentials([string(credentialsId: 'hex-write', variable: 'HEX_API_KEY')]) {
+                image.run('--rm -e HEX_API_KEY=$HEX_API_KEY', 'mix hex.publish --yes')
+            }
+        }
+    }
+}

README.md

@@ -1,2 +1,64 @@
-# divo-kafka
-A pre-configured docker-compose stack for a Kafka cluster for integration testing with Divo
+# DivoKafka
+
+A library implementing the Divo Stack behaviour, providing a pre-configured Kafka
+cluster via docker-compose for integration testing Elixir apps. The cluster is a
+single-node kafka/zookeeper compose stack that can be configured with an arbitrary
+list of topics to create on first start and the hostname/IP address the cluster
+exposes to outside hosts.
+
+Requires inclusion of the Divo library in your mix project.
+
+## Installation
+
+The package can be installed by adding `divo_kafka` to your list of dependencies in `mix.exs`:
+
+```elixir
+def deps do
+  [
+    {:divo, "~> 1.1", organization: "smartcolumbus_os"},
+    {:divo_kafka, "~> 0.1.0", organization: "smartcolumbus_os"}
+  ]
+end
+```
+
+## Use
+
+In your Mix environment exs file (i.e. config/integation.exs), include the following:
+```elixir
+config :myapp,
+  divo: [
+    {DivoKafka, [create_topics: "my-data:1:1", outside_host: "ci-host"]}
+  ]
+```
+
+Then you may use the mix tasks `mix docker.start`, `mix docker.stop`, and `mix docker.kill`
+to manually stand up the stack for debugging or interacting via IEx, or include the
+use directly in your integration tests by adding `use Divo` to the top of any test files
+that will serve for integration testing.
+
+The resulting stack will create a single-node Kafka and Zookeeper instance with
+Zookeeper exposing port 2181 to the host and Kafka exposing port 9092 to the host.
+
+### Configuration
+
+You may omit the configuration arguments to DivoKafka and still have a working stack.
+
+* create_topics: A string of the form `topic1-name:1:1,topic2-name:1:1` which will ensure
+the list of topics are created at first start of the cluster. Defaults to `clusterready:1:1`
+to allow for readiness checking of the cluster.
+
+* outside_host: The hostname or IP address by which hosts external to the Kafka cluster can
+reach it (in this case, your app). Defaults to `localhost` but may cause problems when
+running in a CI system, particularly a containerized one. In such circumstances, it is
+recommended to use an address or name that is routable even if ExUnit will be running inside
+a container.
+
+See [Divo](https://github.com/smartcolumbusos/divo) for more instructions on using and configuring
+the Divo library.
+See [wurstmeister/kafka](https://github.com/wurstmeister/kafka-docker) and
+[wurstmeister/zookeeper](https://github.com/wurstmeister/zookeeper-docker) for further documentation
+on using and configuring the features of these images.
+
+Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc)
+and published on [HexDocs](https://hexdocs.pm). Once published, the docs can
+be found at [https://hexdocs.pm/divo_kafka](https://hexdocs.pm/divo_kafka).

config/config.exs

@@ -0,0 +1,30 @@
+# This file is responsible for configuring your application
+# and its dependencies with the aid of the Mix.Config module.
+use Mix.Config
+
+# This configuration is loaded before any dependency and is restricted
+# to this project. If another project depends on this project, this
+# file won't be loaded nor affect the parent project. For this reason,
+# if you want to provide default values for your application for
+# third-party users, it should be done in your "mix.exs" file.
+
+# You can configure your application as:
+#
+#     config :divo_kafka, key: :value
+#
+# and access this configuration in your application as:
+#
+#     Application.get_env(:divo_kafka, :key)
+#
+# You can also configure a third-party app:
+#
+#     config :logger, level: :info
+#
+
+# It is also possible to import configuration files, relative to this
+# directory. For example, you can emulate configuration per environment
+# by uncommenting the line below and defining dev.exs, test.exs and such.
+# Configuration from the imported file will override the ones defined
+# here (which is why it is important to import them last).
+#
+#     import_config "#{Mix.env()}.exs"

lib/divo_kafka.ex

@@ -0,0 +1,58 @@
+defmodule DivoKafka do
+  @moduledoc """
+  Defines a simple kafka and zookeeper stack as a
+  map compatible with divo for building a docker-compose
+  file.
+  """
+  @behaviour Divo.Stack
+
+  @doc """
+  Implements the Divo Stack behaviour to take a
+  keyword list of defined variables specific to
+  the DivoKafka stack and returns a map describing the
+  service definition of zookeeper and kafka.
+  """
+  @impl Divo.Stack
+  @spec gen_stack([tuple()]) :: map()
+  def gen_stack(envars) do
+    topics = Keyword.get(envars, :create_topics, "clusterready:1:1")
+    host = Keyword.get(envars, :outside_host, "localhost")
+
+    check_topic =
+      topics
+      |> String.split(":")
+      |> List.first()
+
+    %{
+      zookeeper: %{
+        image: "wurstmeister/zookeeper:latest",
+        ports: ["2181:2181"],
+        healthcheck: %{
+          test: ["CMD-SHELL", "echo ruok | nc -w 2 zookeeper 2181"],
+          interval: "5s",
+          timeout: "10s",
+          retries: 3
+        }
+      },
+      kafka: %{
+        image: "wurstmeister/kafka:latest",
+        ports: ["9092:9092"],
+        environment: [
+          "KAFKA_ADVERTISED_LISTENERS=INSIDE://:9094,OUTSIDE://#{host}:9092",
+          "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT",
+          "KAFKA_LISTENERS=INSIDE://:9094,OUTSIDE://:9092",
+          "KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE",
+          "KAFKA_CREATE_TOPICS=#{topics}",
+          "KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181"
+        ],
+        depends_on: ["zookeeper"],
+        healthcheck: %{
+          test: ["CMD-SHELL", "kafka-topics.sh --zookeeper zookeeper:2181 --list | grep #{check_topic} || exit 1"],
+          interval: "10s",
+          timeout: "20s",
+          retries: 3
+        }
+      }
+    }
+  end
+end

mix.exs

@@ -0,0 +1,43 @@
+defmodule DivoKafka.MixProject do
+  use Mix.Project
+
+  def project do
+    [
+      app: :divo_kafka,
+      version: "0.1.0",
+      elixir: "~> 1.8",
+      start_permanent: Mix.env() == :prod,
+      deps: deps(),
+      package: package(),
+      description: description(),
+      source_url: "https://github.com/SmartColumbusOS/divo-kafka"
+    ]
+  end
+
+  def application do
+    [
+      extra_applications: [:logger]
+    ]
+  end
+
+  defp deps do
+    [
+      {:credo, "~> 1.0", only: :dev, runtime: false},
+      {:divo, "~> 1.1", organization: "smartcolumbus_os"},
+      {:ex_doc, "~> 0.19", only: :dev}
+    ]
+  end
+
+  defp description do
+    "A pre-configured kafka docker-compose stack definition for
+    integration testing with the divo library."
+  end
+
+  defp package do
+    [
+      organization: "smartcolumbus_os",
+      licenses: ["AllRightsReserved"],
+      links: %{"GitHub" => "https://github.com/SmartColumbusOS/divo-kafka"}
+    ]
+  end
+end

mix.lock

@@ -0,0 +1,11 @@
+%{
+  "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"},
+  "credo": {:hex, :credo, "1.0.4", "d2214d4cc88c07f54004ffd5a2a27408208841be5eca9f5a72ce9e8e835f7ede", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
+  "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
+  "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"},
+  "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"},
+  "makeup": {:hex, :makeup, "0.8.0", "9cf32aea71c7fe0a4b2e9246c2c4978f9070257e5c9ce6d4a28ec450a839b55f", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
+  "makeup_elixir": {:hex, :makeup_elixir, "0.13.0", "be7a477997dcac2e48a9d695ec730b2d22418292675c75aa2d34ba0909dcdeda", [:mix], [{:makeup, "~> 0.8", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"},
+  "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm"},
+  "patiently": {:hex, :patiently, "0.2.0", "67eb139591e10c4b363ae0198e832552f191c58894731efd3bf124ec4722267a", [:mix], [], "hexpm"},
+}

test/divo_kafka_test.exs

@@ -0,0 +1,78 @@
+defmodule DivoKafkaTest do
+  use ExUnit.Case
+
+  @zookeeper %{
+    zookeeper: %{
+      image: "wurstmeister/zookeeper:latest",
+      ports: ["2181:2181"],
+      healthcheck: %{
+        test: ["CMD-SHELL", "echo ruok | nc -w 2 zookeeper 2181"],
+        interval: "5s",
+        timeout: "10s",
+        retries: 3
+      }
+    }
+  }
+
+  describe "produces a kafka stack map" do
+    test "produces a kafka stack map with no specified environment variables" do
+      expected =
+        %{
+          kafka: %{
+            image: "wurstmeister/kafka:latest",
+            ports: ["9092:9092"],
+            environment: [
+              "KAFKA_ADVERTISED_LISTENERS=INSIDE://:9094,OUTSIDE://localhost:9092",
+              "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT",
+              "KAFKA_LISTENERS=INSIDE://:9094,OUTSIDE://:9092",
+              "KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE",
+              "KAFKA_CREATE_TOPICS=clusterready:1:1",
+              "KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181"
+            ],
+            depends_on: ["zookeeper"],
+            healthcheck: %{
+              test: ["CMD-SHELL", "kafka-topics.sh --zookeeper zookeeper:2181 --list | grep clusterready || exit 1"],
+              interval: "10s",
+              timeout: "20s",
+              retries: 3
+            }
+          }
+        }
+        |> Map.merge(@zookeeper)
+
+      actual = DivoKafka.gen_stack([])
+
+      assert actual == expected
+    end
+
+    test "produces a kafka stack map with supplied environment config" do
+      expected =
+        %{
+          kafka: %{
+            image: "wurstmeister/kafka:latest",
+            ports: ["9092:9092"],
+            environment: [
+              "KAFKA_ADVERTISED_LISTENERS=INSIDE://:9094,OUTSIDE://ci-host:9092",
+              "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT",
+              "KAFKA_LISTENERS=INSIDE://:9094,OUTSIDE://:9092",
+              "KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE",
+              "KAFKA_CREATE_TOPICS=streaming-data:1:1",
+              "KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181"
+            ],
+            depends_on: ["zookeeper"],
+            healthcheck: %{
+              test: ["CMD-SHELL", "kafka-topics.sh --zookeeper zookeeper:2181 --list | grep streaming-data || exit 1"],
+              interval: "10s",
+              timeout: "20s",
+              retries: 3
+            }
+          }
+        }
+        |> Map.merge(@zookeeper)
+
+      actual = DivoKafka.gen_stack(outside_host: "ci-host", create_topics: "streaming-data:1:1")
+
+      assert actual == expected
+    end
+  end
+end

test/test_helper.exs

@@ -0,0 +1 @@
+ExUnit.start()