From 2be7c31bf383479394830098aaad2cc9ab753e04 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Thu, 20 Jul 2023 13:39:25 -0400 Subject: [PATCH 01/16] Switched over to using alpine:3.12 for the base image in dockerfiles. --- Dockerfile | 41 ++++++++++++++++++++++--------------- Dockerfile-nsv | 47 ++++++++++++++++++++++++++----------------- Dockerfile.standalone | 45 ++++++++++++++++++++++++----------------- Dockerfile.testing | 40 +++++++++++++++++++++++++----------- 4 files changed, 108 insertions(+), 65 deletions(-) diff --git a/Dockerfile b/Dockerfile index a6e181b..3594c73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,34 @@ -FROM ubuntu:18.04 +FROM alpine:3.12 USER root WORKDIR /cvdi-stream ENV DEBIAN_FRONTEND=noninteractive -# Add build tools. -RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100 - -# Install cmake. -RUN apt install -y libprotobuf-dev protobuf-compiler -RUN apt install -y cmake - -# Install librdkafka. -RUN apt-get install -y sudo -RUN wget -qO - https://packages.confluent.io/deb/7.3/archive.key | sudo apt-key add - -RUN add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/7.3 stable main" -RUN add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" -RUN apt update -RUN apt-get install -y libsasl2-modules libsasl2-modules-gssapi-mit libsasl2-dev libssl-dev -RUN apt install -y librdkafka-dev +# update the package manager +RUN apk update + +# add build tools +RUN apk add --upgrade --no-cache --virtual .build-deps \ + build-base \ + cmake \ + git \ + wget \ + gcc \ + g++ \ + linux-headers \ + make \ + musl-dev \ + openssl-dev \ + protobuf-dev \ + protobuf-c-dev \ + sudo \ + bash + +# add librdkafka from edge branch +RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ + && apk upgrade --update-cache --available \ + && apk add --no-cache librdkafka librdkafka-dev # add the source and build files ADD CMakeLists.txt /cvdi-stream diff --git a/Dockerfile-nsv b/Dockerfile-nsv index c9b796e..93e6946 100644 --- a/Dockerfile-nsv +++ b/Dockerfile-nsv @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM alpine:3.12 USER root ARG PPM_CONFIG_FILE ARG PPM_MAP_FILE @@ -7,21 +7,30 @@ WORKDIR /cvdi-stream ENV DEBIAN_FRONTEND=noninteractive -# Add build tools. -RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100 - -# Install cmake. -RUN apt install -y libprotobuf-dev protobuf-compiler -RUN apt install -y cmake - -# Install librdkafka. -RUN apt-get install -y sudo -RUN wget -qO - https://packages.confluent.io/deb/7.3/archive.key | sudo apt-key add - -RUN add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/7.3 stable main" -RUN add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" -RUN apt update -RUN apt-get install -y libsasl2-modules libsasl2-modules-gssapi-mit libsasl2-dev libssl-dev -RUN apt install -y librdkafka-dev +# update the package manager +RUN apk update + +# add build tools +RUN apk add --upgrade --no-cache --virtual .build-deps \ + build-base \ + cmake \ + git \ + wget \ + gcc \ + g++ \ + linux-headers \ + make \ + musl-dev \ + openssl-dev \ + protobuf-dev \ + protobuf-c-dev \ + sudo \ + bash + +# add librdkafka from edge branch +RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ + && apk upgrade --update-cache --available \ + && apk add --no-cache librdkafka librdkafka-dev # add the source and build files ADD CMakeLists.txt /cvdi-stream @@ -34,11 +43,11 @@ ADD config/${PPM_CONFIG_FILE} /cvdi-stream/config/ ADD data/${PPM_MAP_FILE} /cvdi-stream/config/ ADD ./config /cvdi-stream/config -# Do the build. +# do the build RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make -# Add test data. This changes frequently so keep it low in the file. +# add test data (this changes frequently so keep it low in the file) ADD ./docker-test /cvdi-stream/docker-test -# Run the tool. +# run the tool CMD ["/cvdi-stream/docker-test/ppm-nsv.sh"] diff --git a/Dockerfile.standalone b/Dockerfile.standalone index 3c29b2d..c3b2bfc 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -1,25 +1,34 @@ -FROM ubuntu:18.04 +FROM alpine:3.12 USER root WORKDIR /cvdi-stream ENV DEBIAN_FRONTEND=noninteractive -# Add build tools. -RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100 - -# Install cmake. -RUN apt install -y libprotobuf-dev protobuf-compiler -RUN apt install -y cmake - -# Install librdkafka. -RUN apt-get install -y sudo -RUN wget -qO - https://packages.confluent.io/deb/7.3/archive.key | sudo apt-key add - -RUN add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/7.3 stable main" -RUN add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" -RUN apt update -RUN apt-get install -y libsasl2-modules libsasl2-modules-gssapi-mit libsasl2-dev libssl-dev -RUN apt install -y librdkafka-dev +# update the package manager +RUN apk update + +# add build tools +RUN apk add --upgrade --no-cache --virtual .build-deps \ + build-base \ + cmake \ + git \ + wget \ + gcc \ + g++ \ + linux-headers \ + make \ + musl-dev \ + openssl-dev \ + protobuf-dev \ + protobuf-c-dev \ + sudo \ + bash + +# add librdkafka from edge branch +RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ + && apk upgrade --update-cache --available \ + && apk add --no-cache librdkafka librdkafka-dev # add the source and build files ADD CMakeLists.txt /cvdi-stream @@ -30,8 +39,8 @@ ADD ./kafka-test /cvdi-stream/kafka-test ADD ./unit-test-data /cvdi-stream/unit-test-data ADD ./config /cvdi-stream/config -# Do the build. +# do the build RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make -# Add test data. This changes frequently so keep it low in the file. +# add test data (this changes frequently so keep it low in the file) ADD ./docker-test /cvdi-stream/docker-test diff --git a/Dockerfile.testing b/Dockerfile.testing index 47c14f3..dca9661 100644 --- a/Dockerfile.testing +++ b/Dockerfile.testing @@ -1,22 +1,38 @@ -FROM ubuntu:18.04 +FROM alpine:3.12 USER root WORKDIR /cvdi-stream ENV DEBIAN_FRONTEND=noninteractive -# Add build tools. -RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100 +# update the package manager +RUN apk update + +# add build tools +RUN apk add --upgrade --no-cache --virtual .build-deps \ + build-base \ + cmake \ + git \ + wget \ + gcc \ + g++ \ + linux-headers \ + make \ + musl-dev \ + openssl-dev \ + protobuf-dev \ + protobuf-c-dev \ + sudo \ + bash + +# add librdkafka from edge branch +RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ + && apk upgrade --update-cache --available \ + && apk add --no-cache librdkafka librdkafka-dev + +# add python3 for testing +RUN apk add --upgrade --no-cache python3 python3-dev -# Install cmake. -RUN apt install -y libprotobuf-dev protobuf-compiler -RUN apt install -y cmake - -# Install librdkafka. -RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev - -# install python for testing -RUN apt-get install -y python3 # add the source and build files ADD CMakeLists.txt /cvdi-stream From db60a284912a0df23bbab6fa4e1a5d7416cd9a19 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Mon, 24 Jul 2023 10:51:23 -0600 Subject: [PATCH 02/16] Split dockerfile contents into BUILDER & RUNTIME images. --- Dockerfile | 30 +++++++++++++++++++++++++++--- Dockerfile-nsv | 28 ++++++++++++++++++++++++++-- Dockerfile.standalone | 28 ++++++++++++++++++++++++++-- Dockerfile.testing | 30 +++++++++++++++++++++++++++--- 4 files changed, 106 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3594c73..85af067 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM alpine:3.12 +# === BUILDER IMAGE === +FROM alpine:3.12 as builder USER root WORKDIR /cvdi-stream @@ -8,7 +9,7 @@ ENV DEBIAN_FRONTEND=noninteractive # update the package manager RUN apk update -# add build tools +# add build dependencies RUN apk add --upgrade --no-cache --virtual .build-deps \ build-base \ cmake \ @@ -39,9 +40,32 @@ ADD ./kafka-test /cvdi-stream/kafka-test ADD ./unit-test-data /cvdi-stream/unit-test-data ADD ./config /cvdi-stream/config -# Do the build. +# do the build RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make +# === RUNTIME IMAGE === +FROM alpine:3.12 +USER root + +WORKDIR /cvdi-stream + +# add runtime dependencies +RUN apk add --upgrade --no-cache \ + bash \ + libstdc++ \ + libgcc \ + libprotobuf \ + protobuf + +# add librdkafka from edge branch +RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ + && apk upgrade --update-cache --available \ + && apk add --no-cache librdkafka librdkafka-dev + +# copy the built files from the builder +COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/ +COPY --from=builder /cvdi-stream /cvdi-stream + # Add test data. This changes frequently so keep it low in the file. ADD ./docker-test /cvdi-stream/docker-test diff --git a/Dockerfile-nsv b/Dockerfile-nsv index 93e6946..f083095 100644 --- a/Dockerfile-nsv +++ b/Dockerfile-nsv @@ -1,4 +1,5 @@ -FROM alpine:3.12 +# === BUILDER IMAGE === +FROM alpine:3.12 as builder USER root ARG PPM_CONFIG_FILE ARG PPM_MAP_FILE @@ -10,7 +11,7 @@ ENV DEBIAN_FRONTEND=noninteractive # update the package manager RUN apk update -# add build tools +# add build dependencies RUN apk add --upgrade --no-cache --virtual .build-deps \ build-base \ cmake \ @@ -46,6 +47,29 @@ ADD ./config /cvdi-stream/config # do the build RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make +# === RUNTIME IMAGE === +FROM alpine:3.12 +USER root + +WORKDIR /cvdi-stream + +# add runtime dependencies +RUN apk add --upgrade --no-cache \ + bash \ + libstdc++ \ + libgcc \ + libprotobuf \ + protobuf + +# add librdkafka from edge branch +RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ + && apk upgrade --update-cache --available \ + && apk add --no-cache librdkafka librdkafka-dev + +# copy the built files from the builder +COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/ +COPY --from=builder /cvdi-stream /cvdi-stream + # add test data (this changes frequently so keep it low in the file) ADD ./docker-test /cvdi-stream/docker-test diff --git a/Dockerfile.standalone b/Dockerfile.standalone index c3b2bfc..94f1f3a 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -1,4 +1,5 @@ -FROM alpine:3.12 +# === BUILDER IMAGE === +FROM alpine:3.12 as builder USER root WORKDIR /cvdi-stream @@ -8,7 +9,7 @@ ENV DEBIAN_FRONTEND=noninteractive # update the package manager RUN apk update -# add build tools +# add build dependencies RUN apk add --upgrade --no-cache --virtual .build-deps \ build-base \ cmake \ @@ -42,5 +43,28 @@ ADD ./config /cvdi-stream/config # do the build RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make +# === RUNTIME IMAGE === +FROM alpine:3.12 +USER root + +WORKDIR /cvdi-stream + +# add runtime dependencies +RUN apk add --upgrade --no-cache \ + bash \ + libstdc++ \ + libgcc \ + libprotobuf \ + protobuf + +# add librdkafka from edge branch +RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ + && apk upgrade --update-cache --available \ + && apk add --no-cache librdkafka librdkafka-dev + +# copy the built files from the builder +COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/ +COPY --from=builder /cvdi-stream /cvdi-stream + # add test data (this changes frequently so keep it low in the file) ADD ./docker-test /cvdi-stream/docker-test diff --git a/Dockerfile.testing b/Dockerfile.testing index dca9661..b2c524f 100644 --- a/Dockerfile.testing +++ b/Dockerfile.testing @@ -1,4 +1,5 @@ -FROM alpine:3.12 +# === BUILDER IMAGE === +FROM alpine:3.12 as builder USER root WORKDIR /cvdi-stream @@ -8,7 +9,7 @@ ENV DEBIAN_FRONTEND=noninteractive # update the package manager RUN apk update -# add build tools +# add build dependencies RUN apk add --upgrade --no-cache --virtual .build-deps \ build-base \ cmake \ @@ -33,7 +34,6 @@ RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ # add python3 for testing RUN apk add --upgrade --no-cache python3 python3-dev - # add the source and build files ADD CMakeLists.txt /cvdi-stream ADD ./src /cvdi-stream/src @@ -46,6 +46,30 @@ ADD ./config /cvdi-stream/config # Do the build. RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make +# === RUNTIME IMAGE === +FROM alpine:3.12 +USER root + +WORKDIR /cvdi-stream + +# add runtime dependencies +RUN apk add --upgrade --no-cache \ + bash \ + libstdc++ \ + libgcc \ + libprotobuf \ + protobuf \ + python3 + +# add librdkafka from edge branch +RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ + && apk upgrade --update-cache --available \ + && apk add --no-cache librdkafka librdkafka-dev + +# copy the built files from the builder +COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/ +COPY --from=builder /cvdi-stream /cvdi-stream + # Add test data. This changes frequently so keep it low in the file. ADD ./docker-test /cvdi-stream/docker-test From 312ec3e1bd4ff53c75f154f3c5064a88807c1896 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Tue, 25 Jul 2023 09:52:21 -0600 Subject: [PATCH 03/16] Removed unnecessary packages from dockerfiles. --- Dockerfile | 19 ++----------------- Dockerfile-nsv | 19 ++----------------- Dockerfile.standalone | 19 ++----------------- Dockerfile.testing | 17 +---------------- 4 files changed, 7 insertions(+), 67 deletions(-) diff --git a/Dockerfile b/Dockerfile index 85af067..8011e29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,20 +11,9 @@ RUN apk update # add build dependencies RUN apk add --upgrade --no-cache --virtual .build-deps \ - build-base \ cmake \ - git \ - wget \ - gcc \ g++ \ - linux-headers \ - make \ - musl-dev \ - openssl-dev \ - protobuf-dev \ - protobuf-c-dev \ - sudo \ - bash + make # add librdkafka from edge branch RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ @@ -51,11 +40,7 @@ WORKDIR /cvdi-stream # add runtime dependencies RUN apk add --upgrade --no-cache \ - bash \ - libstdc++ \ - libgcc \ - libprotobuf \ - protobuf + bash # add librdkafka from edge branch RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ diff --git a/Dockerfile-nsv b/Dockerfile-nsv index f083095..326b9c0 100644 --- a/Dockerfile-nsv +++ b/Dockerfile-nsv @@ -13,20 +13,9 @@ RUN apk update # add build dependencies RUN apk add --upgrade --no-cache --virtual .build-deps \ - build-base \ cmake \ - git \ - wget \ - gcc \ g++ \ - linux-headers \ - make \ - musl-dev \ - openssl-dev \ - protobuf-dev \ - protobuf-c-dev \ - sudo \ - bash + make # add librdkafka from edge branch RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ @@ -55,11 +44,7 @@ WORKDIR /cvdi-stream # add runtime dependencies RUN apk add --upgrade --no-cache \ - bash \ - libstdc++ \ - libgcc \ - libprotobuf \ - protobuf + bash # add librdkafka from edge branch RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ diff --git a/Dockerfile.standalone b/Dockerfile.standalone index 94f1f3a..cb3a697 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -11,20 +11,9 @@ RUN apk update # add build dependencies RUN apk add --upgrade --no-cache --virtual .build-deps \ - build-base \ cmake \ - git \ - wget \ - gcc \ g++ \ - linux-headers \ - make \ - musl-dev \ - openssl-dev \ - protobuf-dev \ - protobuf-c-dev \ - sudo \ - bash + make # add librdkafka from edge branch RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ @@ -51,11 +40,7 @@ WORKDIR /cvdi-stream # add runtime dependencies RUN apk add --upgrade --no-cache \ - bash \ - libstdc++ \ - libgcc \ - libprotobuf \ - protobuf + bash # add librdkafka from edge branch RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ diff --git a/Dockerfile.testing b/Dockerfile.testing index b2c524f..e2cafea 100644 --- a/Dockerfile.testing +++ b/Dockerfile.testing @@ -11,20 +11,9 @@ RUN apk update # add build dependencies RUN apk add --upgrade --no-cache --virtual .build-deps \ - build-base \ cmake \ - git \ - wget \ - gcc \ g++ \ - linux-headers \ - make \ - musl-dev \ - openssl-dev \ - protobuf-dev \ - protobuf-c-dev \ - sudo \ - bash + make # add librdkafka from edge branch RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ @@ -55,10 +44,6 @@ WORKDIR /cvdi-stream # add runtime dependencies RUN apk add --upgrade --no-cache \ bash \ - libstdc++ \ - libgcc \ - libprotobuf \ - protobuf \ python3 # add librdkafka from edge branch From 722a5ca7c02ed89c2f3fd0cb9c24aabe0ef732c8 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Wed, 26 Jul 2023 08:10:30 -0600 Subject: [PATCH 04/16] Installed librdkafka from alpine's stable branch instead of edge. --- Dockerfile | 18 ++++++------------ Dockerfile-nsv | 18 ++++++------------ Dockerfile.standalone | 18 ++++++------------ Dockerfile.testing | 21 ++++++--------------- 4 files changed, 24 insertions(+), 51 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8011e29..1b2cc8e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,12 +13,9 @@ RUN apk update RUN apk add --upgrade --no-cache --virtual .build-deps \ cmake \ g++ \ - make - -# add librdkafka from edge branch -RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ - && apk upgrade --update-cache --available \ - && apk add --no-cache librdkafka librdkafka-dev + make \ + librdkafka \ + librdkafka-dev # add the source and build files ADD CMakeLists.txt /cvdi-stream @@ -40,12 +37,9 @@ WORKDIR /cvdi-stream # add runtime dependencies RUN apk add --upgrade --no-cache \ - bash - -# add librdkafka from edge branch -RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ - && apk upgrade --update-cache --available \ - && apk add --no-cache librdkafka librdkafka-dev + bash \ + librdkafka \ + librdkafka-dev # copy the built files from the builder COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/ diff --git a/Dockerfile-nsv b/Dockerfile-nsv index 326b9c0..4d14caf 100644 --- a/Dockerfile-nsv +++ b/Dockerfile-nsv @@ -15,12 +15,9 @@ RUN apk update RUN apk add --upgrade --no-cache --virtual .build-deps \ cmake \ g++ \ - make - -# add librdkafka from edge branch -RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ - && apk upgrade --update-cache --available \ - && apk add --no-cache librdkafka librdkafka-dev + make \ + librdkafka \ + librdkafka-dev # add the source and build files ADD CMakeLists.txt /cvdi-stream @@ -44,12 +41,9 @@ WORKDIR /cvdi-stream # add runtime dependencies RUN apk add --upgrade --no-cache \ - bash - -# add librdkafka from edge branch -RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ - && apk upgrade --update-cache --available \ - && apk add --no-cache librdkafka librdkafka-dev + bash \ + librdkafka \ + librdkafka-dev # copy the built files from the builder COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/ diff --git a/Dockerfile.standalone b/Dockerfile.standalone index cb3a697..fa6a70d 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -13,12 +13,9 @@ RUN apk update RUN apk add --upgrade --no-cache --virtual .build-deps \ cmake \ g++ \ - make - -# add librdkafka from edge branch -RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ - && apk upgrade --update-cache --available \ - && apk add --no-cache librdkafka librdkafka-dev + make \ + librdkafka \ + librdkafka-dev # add the source and build files ADD CMakeLists.txt /cvdi-stream @@ -40,12 +37,9 @@ WORKDIR /cvdi-stream # add runtime dependencies RUN apk add --upgrade --no-cache \ - bash - -# add librdkafka from edge branch -RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ - && apk upgrade --update-cache --available \ - && apk add --no-cache librdkafka librdkafka-dev + bash \ + librdkafka \ + librdkafka-dev # copy the built files from the builder COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/ diff --git a/Dockerfile.testing b/Dockerfile.testing index e2cafea..4d9e9a0 100644 --- a/Dockerfile.testing +++ b/Dockerfile.testing @@ -13,15 +13,9 @@ RUN apk update RUN apk add --upgrade --no-cache --virtual .build-deps \ cmake \ g++ \ - make - -# add librdkafka from edge branch -RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ - && apk upgrade --update-cache --available \ - && apk add --no-cache librdkafka librdkafka-dev - -# add python3 for testing -RUN apk add --upgrade --no-cache python3 python3-dev + make \ + librdkafka \ + librdkafka-dev # add the source and build files ADD CMakeLists.txt /cvdi-stream @@ -44,12 +38,9 @@ WORKDIR /cvdi-stream # add runtime dependencies RUN apk add --upgrade --no-cache \ bash \ - python3 - -# add librdkafka from edge branch -RUN sed -i -e 's/v3\.4/edge/g' /etc/apk/repositories \ - && apk upgrade --update-cache --available \ - && apk add --no-cache librdkafka librdkafka-dev + python3 \ + librdkafka \ + librdkafka-dev # copy the built files from the builder COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/ From 42d4dcf6f452efbc9920908e0c9285628900cd3c Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Wed, 26 Jul 2023 10:41:51 -0600 Subject: [PATCH 05/16] Refactored a unit test to be OS-independent by decreasing the precision of some output to 7 decimal places. --- src/tests.cpp | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/tests.cpp b/src/tests.cpp index 7eba1c0..5e49724 100644 --- a/src/tests.cpp +++ b/src/tests.cpp @@ -628,10 +628,24 @@ TEST_CASE("Entity", "[quad][entity]") { ss << *phss_area; SECTION("Area") { - // The check below works on os x but not on Ubunutu. - // CHECK(ss.str() == "[35.95255324700415,-83.93236639356081, 35.94893124700415,-83.92801339666028, 35.94882475295794,-83.92814860324864, 35.95244675295795,-83.93250160634807, ]"); - // The check below works on Ubuntu but not os x. - CHECK(ss.str() == "[35.95255324700415,-83.93236639356081, 35.94893124700415,-83.92801339666028, 35.94882475295794,-83.92814860324864, 35.95244675295794,-83.93250160634807, ]"); + // decrease the precision of the output "[x.xxx, x.xxx, x.xxx, ]" for testing + int precision = 7; + std::string copyOfAreaString = ss.str(); + copyOfAreaString.erase(std::remove(copyOfAreaString.begin(), copyOfAreaString.end(), ' '), copyOfAreaString.end()); + for (int currentCharIndex = 0; currentCharIndex < copyOfAreaString.size(); currentCharIndex++) { + if (copyOfAreaString[currentCharIndex] == '.') { + int nextCommaIndex = currentCharIndex + 1; + while (copyOfAreaString[nextCommaIndex] != ',') { + nextCommaIndex++; + } + int offset = currentCharIndex + (precision + 1); + int count = nextCommaIndex - currentCharIndex - (precision + 1); + copyOfAreaString.erase(offset, count); + } + } + std::string expectedOutput = "[35.9525532,-83.9323663,35.9489312,-83.9280133,35.9488247,-83.9281486,35.9524467,-83.9325016,]"; + CHECK(copyOfAreaString == expectedOutput); + CHECK_THROWS(phss->to_area(0.0, 10)); CHECK_THROWS(phss->to_area(-1.0, 10)); CHECK_NOTHROW(phss->to_area(10.0, 5)); From fb203f138a1767cd71ef909e4102765d9e491207 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Fri, 10 Nov 2023 10:08:02 -0700 Subject: [PATCH 06/16] Added image documentation to `docs/dockerhub.md` --- docs/dockerhub.md | 80 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 docs/dockerhub.md diff --git a/docs/dockerhub.md b/docs/dockerhub.md new file mode 100644 index 0000000..931d8a3 --- /dev/null +++ b/docs/dockerhub.md @@ -0,0 +1,80 @@ +# jpo-cvdp + +## GitHub Repository Link +https://github.com/usdot-jpo-ode/jpo-cvdp + +## Purpose +The purpose of the jpo-cvdp program is to filter messages based on location/speed and redact personal identifiable information (PII) from processed messages. + +## How to pull the latest image +The latest image can be pulled using the following command: +> docker pull usdotjpoode/jpo-cvdp:develop + +## Required environment variables +The image expects the following environment variables to be set: +- DOCKER_HOST_IP +- DOCKER_SHARED_VOLUME +- PPM_CONFIG_FILE +- REDACTION_PROPERTIES_PATH + +## Required files in `ppm_data` mounted directory +- fieldsToRedact.txt +- I_80.edges +- ppmBsm.properties + +## Direct Dependencies +The image will fail to start up if the following containers are not already present: +- Kafka +- Zookeeper (relied on by Kafka) + +## Indirect Dependencies +The image will not receive messages to process if the ODE is not running. + +## Example docker-compose.yml with direct dependencies: +``` +version: '2' +services: + zookeeper: + image: wurstmeister/zookeeper + ports: + - "2181:2181" + + kafka: + image: wurstmeister/kafka + ports: + - "9092:9092" + environment: + KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP} + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CREATE_TOPICS: "topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1,topic.OdeTimJson:1:1,topic.FilteredOdeTimJson:1:1" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + + ppm: + image: usdotjpoode/jpo-cvdp:release_q3 + environment: + # required + DOCKER_HOST_IP: ${DOCKER_HOST_IP} + PPM_CONFIG_FILE: ${PPM_CONFIG_FILE} + REDACTION_PROPERTIES_PATH: ${REDACTION_PROPERTIES_PATH} + # optional + PPM_LOG_TO_FILE: "false" + PPM_LOG_TO_CONSOLE: ${PPM_LOG_TO_CONSOLE} + RPM_DEBUG: "false" + PPM_LOG_LEVEL: ${PPM_LOG_LEVEL} + depends_on: + - kafka + volumes: + - ${DOCKER_SHARED_VOLUME_WINDOWS}:/ppm_data +``` + +## Expected startup output +The latest logs should look like this: +``` +jpo-cvdp-ppm-1 | [231109 22:20:14.700649] [info] Waiting for more BSMs from the ODE producer. +jpo-cvdp-ppm-1 | [231109 22:20:15.201389] [info] Waiting for more BSMs from the ODE producer. +jpo-cvdp-ppm-1 | [231109 22:20:15.701666] [info] Waiting for more BSMs from the ODE producer. +jpo-cvdp-ppm-1 | [231109 22:20:16.202104] [info] Waiting for more BSMs from the ODE producer. +jpo-cvdp-ppm-1 | [231109 22:20:16.702451] [info] Waiting for more BSMs from the ODE producer. +jpo-cvdp-ppm-1 | [231109 22:20:17.202653] [info] Waiting for more BSMs from the ODE producer. +``` From f2c4c24823608b96db875216aaa57ad795f8805b Mon Sep 17 00:00:00 2001 From: Marc Wodahl Date: Thu, 4 Jan 2024 21:26:00 -0700 Subject: [PATCH 07/16] Update do_kafka_test to assign DOCKER_HOST_IP if not set --- do_kafka_test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 0237bbc..806179d 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -22,9 +22,9 @@ PPM_IMAGE_TAG=do-kafka-test-ppm-image PPM_IMAGE_NAME=jpo-cvdp_ppm setup() { - if [ -z $DOCKER_HOST_IP ]; then - echo "DOCKER_HOST_IP is not set. Exiting." - exit 1 + if [ -z $DOCKER_HOST_IP ] + then + export DOCKER_HOST_IP=$(ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1' | grep -v '10.*' | head -n 1) fi # print setup info From 7611da465fdf3220db0d0716ddcef36e6b25bca5 Mon Sep 17 00:00:00 2001 From: Marc Wodahl Date: Fri, 5 Jan 2024 09:50:20 -0700 Subject: [PATCH 08/16] update ifconfig command, add additional check if ip can't be determined --- do_kafka_test.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 806179d..6c5c9fc 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -24,8 +24,14 @@ PPM_IMAGE_NAME=jpo-cvdp_ppm setup() { if [ -z $DOCKER_HOST_IP ] then - export DOCKER_HOST_IP=$(ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1' | grep -v '10.*' | head -n 1) + export DOCKER_HOST_IP=$(ifconfig | grep -A 1 'inet ' | grep -v 'inet6\|127.0.0.1' | awk '{print $2}' | grep -E '^172\.1[6-9]\.|^172\.2[0-9]\.|^172\.3[0-1]\.|^192\.168\.' | head -n 1) fi + if [ -z $DOCKER_HOST_IP ] + then + echo "DOCKER_HOST_IP is not set and could not be determined. Exiting." + exit 1 + fi + # print setup info echo "=== Setup Info ===" From 37e96bf20c2876d2606c06e1ba59f80fe1fc69c9 Mon Sep 17 00:00:00 2001 From: Marc Wodahl Date: Wed, 24 Jan 2024 08:49:33 -0700 Subject: [PATCH 09/16] Update KAFKA_CONTAINER_NAME to resolve dynamically --- do_kafka_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 6c5c9fc..3ddef7f 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -12,8 +12,8 @@ CYAN='\033[0;36m' YELLOW='\033[1;33m' NC='\033[0m' # No Color -CURRENT_DIR_NAME=${PWD##*/} -KAFKA_CONTAINER_NAME=$CURRENT_DIR_NAME-kafka-1 + +KAFKA_CONTAINER_NAME=$(docker ps --format '{{.Names}}' | grep kafka) MAP_FILE=data/I_80.edges BSM_DATA_FILE=data/I_80_test.json TIM_DATA_FILE=data/I_80_test_TIMS.json From 43c7923197d48bdc6aad9b9bd009e5cfeac33336 Mon Sep 17 00:00:00 2001 From: Marc Wodahl Date: Thu, 25 Jan 2024 09:20:51 -0700 Subject: [PATCH 10/16] Move KAFKA_CONTAINER_NAME to waitForKafkaToCreateTopics --- do_kafka_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 3ddef7f..6d7c063 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -13,7 +13,6 @@ YELLOW='\033[1;33m' NC='\033[0m' # No Color -KAFKA_CONTAINER_NAME=$(docker ps --format '{{.Names}}' | grep kafka) MAP_FILE=data/I_80.edges BSM_DATA_FILE=data/I_80_test.json TIM_DATA_FILE=data/I_80_test_TIMS.json @@ -36,7 +35,7 @@ setup() { # print setup info echo "=== Setup Info ===" echo "DOCKER_HOST_IP: $DOCKER_HOST_IP" - echo "KAFKA_CONTAINER_NAME: $KAFKA_CONTAINER_NAME" + echo "KAFKA_CONTAINER_NAME is resolved dynamically" echo "MAP_FILE: $MAP_FILE" echo "BSM_DATA_FILE: $BSM_DATA_FILE" echo "TIM_DATA_FILE: $TIM_DATA_FILE" @@ -51,6 +50,7 @@ setup() { waitForKafkaToCreateTopics() { maxAttempts=100 attempts=0 + KAFKA_CONTAINER_NAME=$(docker ps --format '{{.Names}}' | grep kafka) while true; do attempts=$((attempts+1)) if [ $(docker ps | grep $KAFKA_CONTAINER_NAME | wc -l) == "0" ]; then From 9a4a74988604b5bcbe9b367d9dfa150b660cb01f Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Tue, 30 Jan 2024 08:35:06 -0700 Subject: [PATCH 11/16] Added release notes for v1.3.0 --- docs/Release_notes.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/Release_notes.md b/docs/Release_notes.md index 041926e..6dff78b 100644 --- a/docs/Release_notes.md +++ b/docs/Release_notes.md @@ -1,6 +1,21 @@ Jpo-cvdp Release Notes ---------------------------- +Version 1.3.0, released February 2024 + +### **Summary** +The changes for the jpo-cvdp 1.3.0 release involve the optimization of dockerfiles, addition of dockerhub image documentation & some QoL changes to the `do_kafka_test.sh` script. + +Enhancements in this release: +- CDOT PR 32: Optimized dockerfiles for smaller & faster deployments by switching over to using alpine as the base image. +- CDOT PR 37: Added dockerhub image documentation +- CDOT PR 38: Updated `do_kafka_test.sh` script to set DOCKER_HOST_IP automatically +- CDOT PR 39: Updated `do_kafka_test.sh` script to resolve the KAFKA_CONTAINER_NAME env var dynamically + +Known Issues: +- No known issues at this time. + + Version 1.2.0, released November 2023 ---------------------------------------- From 2c54df3249a6db56f9abc11951b5b20d06d8846a Mon Sep 17 00:00:00 2001 From: Drew Johnston <31270488+drewjj@users.noreply.github.com> Date: Wed, 31 Jan 2024 03:52:00 -0700 Subject: [PATCH 12/16] Update CI to latest values to not fail PR build --- .github/workflows/ci.yml | 6 +++--- .github/workflows/docker.yml | 6 +++--- .github/workflows/dockerhub.yml | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 853b343..23e729c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,11 +43,11 @@ jobs: cmake --build _cmake_build cmake --build _cmake_build --target install - name: Install sonar-scanner and build-wrapper - uses: SonarSource/sonarcloud-github-c-cpp@v1 # This Action Installs sonar cloud and build wrapper to run sonar scan analysis + uses: SonarSource/sonarcloud-github-c-cpp@v2 # This Action Installs sonar cloud and build wrapper to run sonar scan analysis - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 - name: Build and Generate test coverage run: | cd $GITHUB_WORKSPACE diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 04900d7..44b7ecb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -13,8 +13,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml index 32d430c..81f0790 100644 --- a/.github/workflows/dockerhub.yml +++ b/.github/workflows/dockerhub.yml @@ -11,16 +11,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: push: true tags: usdotjpoode/jpo-cvdp:${{ github.ref_name }} From c23ccd9ac162a9b456a6d191a444ff9d013d7dfb Mon Sep 17 00:00:00 2001 From: Drew Johnston <31270488+drewjj@users.noreply.github.com> Date: Wed, 31 Jan 2024 04:04:39 -0700 Subject: [PATCH 13/16] Forgot the sonarscanner version --- .github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23e729c..6deb13a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: cmake --build _cmake_build cmake --build _cmake_build --target install - name: Install sonar-scanner and build-wrapper - uses: SonarSource/sonarcloud-github-c-cpp@v2 # This Action Installs sonar cloud and build wrapper to run sonar scan analysis + uses: sonarsource/sonarcloud-github-c-cpp@v2 # This Action Installs sonar cloud and build wrapper to run sonar scan analysis - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build @@ -72,9 +72,7 @@ jobs: name: jpo-cvdp path: /home/runner/work/jpo-cvdp/jpo-cvdp/bw-output - name: Setup SonarScanner - uses: warchant/setup-sonar-scanner@v4 # This action is used to setup sonar scanner with required versions. - with: - version: 4.8.0.2856 + uses: warchant/setup-sonar-scanner@v7 # This action is used to setup sonar scanner with required versions. - name: Generate sonar properties file run: | cat < /tmp/sonar-scanner.properties From 132dd1339e62f408d8fe25e09829337337d9bc46 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Wed, 31 Jan 2024 14:09:53 -0700 Subject: [PATCH 14/16] Removed docker setup & build/push actions from `ci.yml` --- .github/workflows/ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6deb13a..7982b48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,10 +44,6 @@ jobs: cmake --build _cmake_build --target install - name: Install sonar-scanner and build-wrapper uses: sonarsource/sonarcloud-github-c-cpp@v2 # This Action Installs sonar cloud and build wrapper to run sonar scan analysis - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Build - uses: docker/build-push-action@v5 - name: Build and Generate test coverage run: | cd $GITHUB_WORKSPACE From d83423800c9426fcc544f87850ac21a2beff3ec2 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Wed, 31 Jan 2024 14:38:13 -0700 Subject: [PATCH 15/16] Modified dockerfiles to reduce package installation duplication --- Dockerfile | 32 ++++++++++++++++++-------------- Dockerfile-nsv | 32 ++++++++++++++++++-------------- Dockerfile.standalone | 32 ++++++++++++++++++-------------- Dockerfile.testing | 33 ++++++++++++++++++--------------- 4 files changed, 72 insertions(+), 57 deletions(-) diff --git a/Dockerfile b/Dockerfile index 18dc3b5..11d35b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,31 @@ -# === BUILDER IMAGE === -FROM alpine:3.12 as builder +# === RUNTIME DEPENDENCIES IMAGE === +FROM alpine:3.12 as runtime-deps USER root WORKDIR /cvdi-stream -ENV DEBIAN_FRONTEND=noninteractive - # update the package manager RUN apk update +# add runtime dependencies +RUN apk add --upgrade --no-cache \ + bash \ + librdkafka \ + librdkafka-dev + +# === BUILDER IMAGE === +FROM runtime-deps as builder +USER root + +WORKDIR /cvdi-stream + +ENV DEBIAN_FRONTEND=noninteractive + # add build dependencies RUN apk add --upgrade --no-cache --virtual .build-deps \ cmake \ g++ \ - make \ - librdkafka \ - librdkafka-dev + make # add the source and build files ADD CMakeLists.txt /cvdi-stream @@ -30,17 +40,11 @@ ADD ./config /cvdi-stream/config RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make # === RUNTIME IMAGE === -FROM alpine:3.12 +FROM runtime-deps USER root WORKDIR /cvdi-stream -# add runtime dependencies -RUN apk add --upgrade --no-cache \ - bash \ - librdkafka \ - librdkafka-dev - # copy the built files from the builder COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/ COPY --from=builder /cvdi-stream /cvdi-stream diff --git a/Dockerfile-nsv b/Dockerfile-nsv index 4d14caf..28c3139 100644 --- a/Dockerfile-nsv +++ b/Dockerfile-nsv @@ -1,5 +1,20 @@ +# === RUNTIME DEPENDENCIES IMAGE === +FROM alpine:3.12 as runtime-deps +USER root + +WORKDIR /cvdi-stream + +# update the package manager +RUN apk update + +# add runtime dependencies +RUN apk add --upgrade --no-cache \ + bash \ + librdkafka \ + librdkafka-dev + # === BUILDER IMAGE === -FROM alpine:3.12 as builder +FROM runtime-deps as builder USER root ARG PPM_CONFIG_FILE ARG PPM_MAP_FILE @@ -8,16 +23,11 @@ WORKDIR /cvdi-stream ENV DEBIAN_FRONTEND=noninteractive -# update the package manager -RUN apk update - # add build dependencies RUN apk add --upgrade --no-cache --virtual .build-deps \ cmake \ g++ \ - make \ - librdkafka \ - librdkafka-dev + make # add the source and build files ADD CMakeLists.txt /cvdi-stream @@ -34,17 +44,11 @@ ADD ./config /cvdi-stream/config RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make # === RUNTIME IMAGE === -FROM alpine:3.12 +FROM runtime-deps USER root WORKDIR /cvdi-stream -# add runtime dependencies -RUN apk add --upgrade --no-cache \ - bash \ - librdkafka \ - librdkafka-dev - # copy the built files from the builder COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/ COPY --from=builder /cvdi-stream /cvdi-stream diff --git a/Dockerfile.standalone b/Dockerfile.standalone index fa6a70d..bc786cd 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -1,21 +1,31 @@ -# === BUILDER IMAGE === -FROM alpine:3.12 as builder +# === RUNTIME DEPENDENCIES IMAGE === +FROM alpine:3.12 as runtime-deps USER root WORKDIR /cvdi-stream -ENV DEBIAN_FRONTEND=noninteractive - # update the package manager RUN apk update +# add runtime dependencies +RUN apk add --upgrade --no-cache \ + bash \ + librdkafka \ + librdkafka-dev + +# === BUILDER IMAGE === +FROM runtime-deps as builder +USER root + +WORKDIR /cvdi-stream + +ENV DEBIAN_FRONTEND=noninteractive + # add build dependencies RUN apk add --upgrade --no-cache --virtual .build-deps \ cmake \ g++ \ - make \ - librdkafka \ - librdkafka-dev + make # add the source and build files ADD CMakeLists.txt /cvdi-stream @@ -30,17 +40,11 @@ ADD ./config /cvdi-stream/config RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make # === RUNTIME IMAGE === -FROM alpine:3.12 +FROM runtime-deps USER root WORKDIR /cvdi-stream -# add runtime dependencies -RUN apk add --upgrade --no-cache \ - bash \ - librdkafka \ - librdkafka-dev - # copy the built files from the builder COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/ COPY --from=builder /cvdi-stream /cvdi-stream diff --git a/Dockerfile.testing b/Dockerfile.testing index 6a68b5f..f24d3ad 100644 --- a/Dockerfile.testing +++ b/Dockerfile.testing @@ -1,21 +1,31 @@ -# === BUILDER IMAGE === -FROM alpine:3.12 as builder +# === RUNTIME DEPENDENCIES IMAGE === +FROM alpine:3.12 as runtime-deps USER root WORKDIR /cvdi-stream -ENV DEBIAN_FRONTEND=noninteractive - # update the package manager RUN apk update +# add runtime dependencies +RUN apk add --upgrade --no-cache \ + bash \ + librdkafka \ + librdkafka-dev + +# === BUILDER IMAGE === +FROM runtime-deps as builder +USER root + +WORKDIR /cvdi-stream + +ENV DEBIAN_FRONTEND=noninteractive + # add build dependencies RUN apk add --upgrade --no-cache --virtual .build-deps \ cmake \ g++ \ - make \ - librdkafka \ - librdkafka-dev + make # add the source and build files ADD CMakeLists.txt /cvdi-stream @@ -30,18 +40,11 @@ ADD ./config /cvdi-stream/config RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make # === RUNTIME IMAGE === -FROM alpine:3.12 +FROM runtime-deps USER root WORKDIR /cvdi-stream -# add runtime dependencies -RUN apk add --upgrade --no-cache \ - bash \ - python3 \ - librdkafka \ - librdkafka-dev - # copy the built files from the builder COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/ COPY --from=builder /cvdi-stream /cvdi-stream From 56da6dd28eaad355ba536963126dd680005d2d2e Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Wed, 31 Jan 2024 14:46:31 -0700 Subject: [PATCH 16/16] Added python3 runtime dependency back into `Dockerfile.testing` --- Dockerfile.testing | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.testing b/Dockerfile.testing index f24d3ad..572cce6 100644 --- a/Dockerfile.testing +++ b/Dockerfile.testing @@ -10,6 +10,7 @@ RUN apk update # add runtime dependencies RUN apk add --upgrade --no-cache \ bash \ + python3 \ librdkafka \ librdkafka-dev