Unify Docker Kafka used for components (#383)

.github/workflows/ci.yml

@@ -25,28 +25,34 @@ jobs:
             coverage: 'true'
     steps:
       - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
+
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
+
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{matrix.ruby}}
-      - name: Remove libzstd-dev to check no supported compressions
-        run: sudo apt-get -y remove libzstd-dev
+          bundler-cache: true
+
       - name: Run Kafka with docker-compose
-        # We need to give Kafka enough time to start and create all the needed topics, etc
-        # If anyone has a better idea on how to do it smart and easily, please contact me
         run: |
           docker-compose up -d
+
+      - name: Wait for Kafka
+        run: |
           sleep 5
+
       - name: Install latest bundler
         run: |
           gem install bundler --no-document
           bundle config set without 'tools benchmarks docs'
+
       - name: Bundle install
         run: |
           bundle config set without development
           bundle install --jobs 4 --retry 3
+
       - name: Run all tests
         env:
           GITHUB_COVERAGE: ${{matrix.coverage}}

docker-compose.yml

@@ -1,18 +1,24 @@
 version: '2'
+
 services:
   zookeeper:
-    image: wurstmeister/zookeeper
-    ports:
-      - "2181:2181"
+    container_name: waterdrop_zookeeper
+    image: confluentinc/cp-zookeeper:7.5.0
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 100
+
   kafka:
-    image: wurstmeister/kafka:2.12-2.5.0
+    container_name: waterdrop_kafka
+    image: confluentinc/cp-kafka:7.5.0
+    depends_on:
+      - zookeeper
     ports:
-      - "9092:9092"
+      - 9092:9092
     environment:
-      KAFKA_ADVERTISED_HOST_NAME: localhost
-      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
-      KAFKA_CREATE_TOPICS: 'example_topic:1:1'
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

spec/lib/waterdrop/instrumentation/callbacks/delivery_spec.rb

@@ -56,7 +56,7 @@
 
     context 'when there is a message that was not successfully delivered async' do
       let(:changed) { [] }
-      let(:event) { changed.first }
+      let(:event) { changed.last }
 
       before do
         producer.monitor.subscribe('error.occurred') do |event|

spec/lib/waterdrop/producer/sync_spec.rb

@@ -112,10 +112,10 @@
       it { expect(delivery).to be_a(Rdkafka::Producer::DeliveryReport) }
     end
 
-    context 'when it is not installed zstd' do
+    context 'when it is installed zstd' do
       let(:codec) { 'zstd' }
 
-      it { expect { delivery }.to raise_error(Rdkafka::Config::ConfigError) }
+      it { expect(delivery).to be_a(Rdkafka::Producer::DeliveryReport) }
     end
 
     context 'when it is installed lz4' do