Merge pull request #38 from usdot-jpo-ode/candidate_r1

Merge candidate_r1 into master
usdot-jpo-ode · Feb 26, 2024 · e51d334 · e51d334
2 parents a57f0c6 + c22b9d3
commit e51d334
Show file tree

Hide file tree

Showing 11 changed files with 264 additions and 87 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -43,11 +43,7 @@ jobs:
          cmake --build _cmake_build
          cmake --build _cmake_build --target install  
      - name: Install sonar-scanner and build-wrapper
-       uses: SonarSource/sonarcloud-github-c-cpp@v1  # This Action Installs sonar cloud and build wrapper to run sonar scan analysis  
-     - name: Set up Docker Buildx
-       uses: docker/setup-buildx-action@v2
-     - name: Build
-       uses: docker/build-push-action@v3       
+       uses: sonarsource/sonarcloud-github-c-cpp@v2  # This Action Installs sonar cloud and build wrapper to run sonar scan analysis  
      - name: Build and Generate test coverage
        run: |
          cd $GITHUB_WORKSPACE
@@ -72,9 +68,7 @@ jobs:
          name: jpo-cvdp
          path: /home/runner/work/jpo-cvdp/jpo-cvdp/bw-output       
      - name: Setup SonarScanner
-       uses: warchant/setup-sonar-scanner@v4 # This action is used to setup sonar scanner with required versions.
-       with:
-         version: 4.8.0.2856
+       uses: warchant/setup-sonar-scanner@v7 # This action is used to setup sonar scanner with required versions.
      - name: Generate sonar properties file
        run: |
          cat <<EOF > /tmp/sonar-scanner.properties

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -13,8 +13,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml
@@ -11,16 +11,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           push: true
           tags: usdotjpoode/jpo-cvdp:${{ github.ref_name }}
diff --git a/Dockerfile b/Dockerfile
@@ -1,25 +1,31 @@
-FROM ubuntu:18.04
+# === RUNTIME DEPENDENCIES IMAGE ===
+FROM alpine:3.12 as runtime-deps
 USER root
 
 WORKDIR /cvdi-stream
 
-ENV DEBIAN_FRONTEND=noninteractive
+# update the package manager
+RUN apk update
+
+# add runtime dependencies
+RUN apk add --upgrade --no-cache \
+    bash \
+    librdkafka \
+    librdkafka-dev
+
+# === BUILDER IMAGE ===
+FROM runtime-deps as builder
+USER root
 
-# Add build tools.
-RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
+WORKDIR /cvdi-stream
 
-# Install cmake.
-RUN apt install -y libprotobuf-dev protobuf-compiler
-RUN apt install -y cmake
+ENV DEBIAN_FRONTEND=noninteractive
 
-# Install librdkafka.
-RUN apt-get install -y sudo
-RUN wget -qO - https://packages.confluent.io/deb/7.3/archive.key | sudo apt-key add -
-RUN add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/7.3 stable main"
-RUN add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main"
-RUN apt update
-RUN apt-get install -y libsasl2-modules libsasl2-modules-gssapi-mit libsasl2-dev libssl-dev 
-RUN apt install -y librdkafka-dev
+# add build dependencies
+RUN apk add --upgrade --no-cache --virtual .build-deps \
+    cmake \
+    g++ \
+    make
 
 # add the source and build files
 ADD CMakeLists.txt /cvdi-stream
@@ -30,9 +36,19 @@ ADD ./kafka-test /cvdi-stream/kafka-test
 ADD ./unit-test-data /cvdi-stream/unit-test-data
 ADD ./config /cvdi-stream/config
 
-# Do the build.
+# do the build
 RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make
 
+# === RUNTIME IMAGE ===
+FROM runtime-deps
+USER root
+
+WORKDIR /cvdi-stream
+
+# copy the built files from the builder
+COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/
+COPY --from=builder /cvdi-stream /cvdi-stream
+
 # Add test data. This changes frequently so keep it low in the file.
 ADD ./docker-test /cvdi-stream/docker-test
 

diff --git a/Dockerfile-nsv b/Dockerfile-nsv
@@ -1,4 +1,20 @@
-FROM ubuntu:18.04
+# === RUNTIME DEPENDENCIES IMAGE ===
+FROM alpine:3.12 as runtime-deps
+USER root
+
+WORKDIR /cvdi-stream
+
+# update the package manager
+RUN apk update
+
+# add runtime dependencies
+RUN apk add --upgrade --no-cache \
+    bash \
+    librdkafka \
+    librdkafka-dev
+
+# === BUILDER IMAGE ===
+FROM runtime-deps as builder
 USER root
 ARG PPM_CONFIG_FILE
 ARG PPM_MAP_FILE
@@ -7,21 +23,11 @@ WORKDIR /cvdi-stream
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-# Add build tools.
-RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
-
-# Install cmake.
-RUN apt install -y libprotobuf-dev protobuf-compiler
-RUN apt install -y cmake
-
-# Install librdkafka.
-RUN apt-get install -y sudo
-RUN wget -qO - https://packages.confluent.io/deb/7.3/archive.key | sudo apt-key add -
-RUN add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/7.3 stable main"
-RUN add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main"
-RUN apt update
-RUN apt-get install -y libsasl2-modules libsasl2-modules-gssapi-mit libsasl2-dev libssl-dev 
-RUN apt install -y librdkafka-dev
+# add build dependencies
+RUN apk add --upgrade --no-cache --virtual .build-deps \
+    cmake \
+    g++ \
+    make
 
 # add the source and build files
 ADD CMakeLists.txt /cvdi-stream
@@ -34,11 +40,21 @@ ADD config/${PPM_CONFIG_FILE} /cvdi-stream/config/
 ADD data/${PPM_MAP_FILE} /cvdi-stream/config/
 ADD ./config /cvdi-stream/config
 
-# Do the build.
+# do the build
 RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make
 
-# Add test data. This changes frequently so keep it low in the file.
+# === RUNTIME IMAGE ===
+FROM runtime-deps
+USER root
+
+WORKDIR /cvdi-stream
+
+# copy the built files from the builder
+COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/
+COPY --from=builder /cvdi-stream /cvdi-stream
+
+# add test data (this changes frequently so keep it low in the file)
 ADD ./docker-test /cvdi-stream/docker-test
 
-# Run the tool.
+# run the tool
 CMD ["/cvdi-stream/docker-test/ppm-nsv.sh"]
diff --git a/Dockerfile.standalone b/Dockerfile.standalone
@@ -1,25 +1,31 @@
-FROM ubuntu:18.04
+# === RUNTIME DEPENDENCIES IMAGE ===
+FROM alpine:3.12 as runtime-deps
 USER root
 
 WORKDIR /cvdi-stream
 
-ENV DEBIAN_FRONTEND=noninteractive
+# update the package manager
+RUN apk update
+
+# add runtime dependencies
+RUN apk add --upgrade --no-cache \
+    bash \
+    librdkafka \
+    librdkafka-dev
+
+# === BUILDER IMAGE ===
+FROM runtime-deps as builder
+USER root
 
-# Add build tools.
-RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
+WORKDIR /cvdi-stream
 
-# Install cmake.
-RUN apt install -y libprotobuf-dev protobuf-compiler
-RUN apt install -y cmake
+ENV DEBIAN_FRONTEND=noninteractive
 
-# Install librdkafka.
-RUN apt-get install -y sudo
-RUN wget -qO - https://packages.confluent.io/deb/7.3/archive.key | sudo apt-key add -
-RUN add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/7.3 stable main"
-RUN add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main"
-RUN apt update
-RUN apt-get install -y libsasl2-modules libsasl2-modules-gssapi-mit libsasl2-dev libssl-dev 
-RUN apt install -y librdkafka-dev
+# add build dependencies
+RUN apk add --upgrade --no-cache --virtual .build-deps \
+    cmake \
+    g++ \
+    make
 
 # add the source and build files
 ADD CMakeLists.txt /cvdi-stream
@@ -30,8 +36,18 @@ ADD ./kafka-test /cvdi-stream/kafka-test
 ADD ./unit-test-data /cvdi-stream/unit-test-data
 ADD ./config /cvdi-stream/config
 
-# Do the build.
+# do the build
 RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make
 
-# Add test data. This changes frequently so keep it low in the file.
+# === RUNTIME IMAGE ===
+FROM runtime-deps
+USER root
+
+WORKDIR /cvdi-stream
+
+# copy the built files from the builder
+COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/
+COPY --from=builder /cvdi-stream /cvdi-stream
+
+# add test data (this changes frequently so keep it low in the file)
 ADD ./docker-test /cvdi-stream/docker-test
diff --git a/Dockerfile.testing b/Dockerfile.testing
@@ -1,22 +1,32 @@
-FROM ubuntu:18.04
+# === RUNTIME DEPENDENCIES IMAGE ===
+FROM alpine:3.12 as runtime-deps
 USER root
 
 WORKDIR /cvdi-stream
 
-ENV DEBIAN_FRONTEND=noninteractive
+# update the package manager
+RUN apk update
 
-# Add build tools.
-RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
+# add runtime dependencies
+RUN apk add --upgrade --no-cache \
+    bash \
+    python3 \
+    librdkafka \
+    librdkafka-dev
 
-# Install cmake.
-RUN apt install -y libprotobuf-dev protobuf-compiler
-RUN apt install -y cmake
+# === BUILDER IMAGE ===
+FROM runtime-deps as builder
+USER root
 
-# Install librdkafka.
-RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev
+WORKDIR /cvdi-stream
+
+ENV DEBIAN_FRONTEND=noninteractive
 
-# install python for testing
-RUN apt-get install -y python3
+# add build dependencies
+RUN apk add --upgrade --no-cache --virtual .build-deps \
+    cmake \
+    g++ \
+    make
 
 # add the source and build files
 ADD CMakeLists.txt /cvdi-stream
@@ -30,6 +40,16 @@ ADD ./config /cvdi-stream/config
 # Do the build.
 RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make
 
+# === RUNTIME IMAGE ===
+FROM runtime-deps
+USER root
+
+WORKDIR /cvdi-stream
+
+# copy the built files from the builder
+COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/
+COPY --from=builder /cvdi-stream /cvdi-stream
+
 # Add test data. This changes frequently so keep it low in the file.
 ADD ./docker-test /cvdi-stream/docker-test
 

diff --git a/do_kafka_test.sh b/do_kafka_test.sh
@@ -12,8 +12,7 @@ CYAN='\033[0;36m'
 YELLOW='\033[1;33m'
 NC='\033[0m' # No Color
 
-CURRENT_DIR_NAME=${PWD##*/}
-KAFKA_CONTAINER_NAME=$CURRENT_DIR_NAME-kafka-1
+
 MAP_FILE=data/I_80.edges
 BSM_DATA_FILE=data/I_80_test.json
 TIM_DATA_FILE=data/I_80_test_TIMS.json
@@ -22,15 +21,21 @@ PPM_IMAGE_TAG=do-kafka-test-ppm-image
 PPM_IMAGE_NAME=jpo-cvdp_ppm
 
 setup() {
-    if [ -z $DOCKER_HOST_IP ]; then
-        echo "DOCKER_HOST_IP is not set. Exiting."
+    if [ -z $DOCKER_HOST_IP ]
+    then
+        export DOCKER_HOST_IP=$(ifconfig | grep -A 1 'inet ' | grep -v 'inet6\|127.0.0.1' | awk '{print $2}' | grep -E '^172\.1[6-9]\.|^172\.2[0-9]\.|^172\.3[0-1]\.|^192\.168\.' | head -n 1)
+    fi
+    if [ -z $DOCKER_HOST_IP ]
+    then
+        echo "DOCKER_HOST_IP is not set and could not be determined. Exiting."
         exit 1
     fi
 
+
     # print setup info
     echo "=== Setup Info ==="
     echo "DOCKER_HOST_IP: $DOCKER_HOST_IP"
-    echo "KAFKA_CONTAINER_NAME: $KAFKA_CONTAINER_NAME"
+    echo "KAFKA_CONTAINER_NAME is resolved dynamically"
     echo "MAP_FILE: $MAP_FILE"
     echo "BSM_DATA_FILE: $BSM_DATA_FILE"
     echo "TIM_DATA_FILE: $TIM_DATA_FILE"
@@ -45,6 +50,7 @@ setup() {
 waitForKafkaToCreateTopics() {
     maxAttempts=100
     attempts=0
+    KAFKA_CONTAINER_NAME=$(docker ps --format '{{.Names}}' | grep kafka)
     while true; do
         attempts=$((attempts+1))
         if [ $(docker ps | grep $KAFKA_CONTAINER_NAME | wc -l) == "0" ]; then

diff --git a/docs/Release_notes.md b/docs/Release_notes.md
@@ -1,6 +1,21 @@
 Jpo-cvdp Release Notes
 ----------------------------
 
+Version 1.3.0, released February 2024
+
+### **Summary**
+The changes for the jpo-cvdp 1.3.0 release involve the optimization of dockerfiles, addition of dockerhub image documentation & some QoL changes to the `do_kafka_test.sh` script.
+
+Enhancements in this release:
+- CDOT PR 32: Optimized dockerfiles for smaller & faster deployments by switching over to using alpine as the base image.
+- CDOT PR 37: Added dockerhub image documentation
+- CDOT PR 38: Updated `do_kafka_test.sh` script to set DOCKER_HOST_IP automatically
+- CDOT PR 39: Updated `do_kafka_test.sh` script to resolve the KAFKA_CONTAINER_NAME env var dynamically
+
+Known Issues:
+- No known issues at this time.
+
+
 Version 1.2.0, released November 2023
 ----------------------------------------