Skip to content

Commit

Permalink
Merge pull request #37 from CDOT-CV/develop
Browse files Browse the repository at this point in the history
Dockerfile Optimization
  • Loading branch information
dan-du-car authored Feb 1, 2024
2 parents 9da5dff + e5e1294 commit c22b9d3
Show file tree
Hide file tree
Showing 11 changed files with 264 additions and 87 deletions.
10 changes: 2 additions & 8 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,7 @@ jobs:
cmake --build _cmake_build
cmake --build _cmake_build --target install
- name: Install sonar-scanner and build-wrapper
uses: SonarSource/sonarcloud-github-c-cpp@v1 # This Action Installs sonar cloud and build wrapper to run sonar scan analysis
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build
uses: docker/build-push-action@v3
uses: sonarsource/sonarcloud-github-c-cpp@v2 # This Action Installs sonar cloud and build wrapper to run sonar scan analysis
- name: Build and Generate test coverage
run: |
cd $GITHUB_WORKSPACE
Expand All @@ -72,9 +68,7 @@ jobs:
name: jpo-cvdp
path: /home/runner/work/jpo-cvdp/jpo-cvdp/bw-output
- name: Setup SonarScanner
uses: warchant/setup-sonar-scanner@v4 # This action is used to setup sonar scanner with required versions.
with:
version: 4.8.0.2856
uses: warchant/setup-sonar-scanner@v7 # This action is used to setup sonar scanner with required versions.
- name: Generate sonar properties file
run: |
cat <<EOF > /tmp/sonar-scanner.properties
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
- name: Build
uses: docker/build-push-action@v3
uses: docker/build-push-action@v5
8 changes: 4 additions & 4 deletions .github/workflows/dockerhub.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,16 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build
uses: docker/build-push-action@v3
uses: docker/build-push-action@v5
with:
push: true
tags: usdotjpoode/jpo-cvdp:${{ github.ref_name }}
48 changes: 32 additions & 16 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,25 +1,31 @@
FROM ubuntu:18.04
# === RUNTIME DEPENDENCIES IMAGE ===
FROM alpine:3.12 as runtime-deps
USER root

WORKDIR /cvdi-stream

ENV DEBIAN_FRONTEND=noninteractive
# update the package manager
RUN apk update

# add runtime dependencies
RUN apk add --upgrade --no-cache \
bash \
librdkafka \
librdkafka-dev

# === BUILDER IMAGE ===
FROM runtime-deps as builder
USER root

# Add build tools.
RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
WORKDIR /cvdi-stream

# Install cmake.
RUN apt install -y libprotobuf-dev protobuf-compiler
RUN apt install -y cmake
ENV DEBIAN_FRONTEND=noninteractive

# Install librdkafka.
RUN apt-get install -y sudo
RUN wget -qO - https://packages.confluent.io/deb/7.3/archive.key | sudo apt-key add -
RUN add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/7.3 stable main"
RUN add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main"
RUN apt update
RUN apt-get install -y libsasl2-modules libsasl2-modules-gssapi-mit libsasl2-dev libssl-dev
RUN apt install -y librdkafka-dev
# add build dependencies
RUN apk add --upgrade --no-cache --virtual .build-deps \
cmake \
g++ \
make

# add the source and build files
ADD CMakeLists.txt /cvdi-stream
Expand All @@ -30,9 +36,19 @@ ADD ./kafka-test /cvdi-stream/kafka-test
ADD ./unit-test-data /cvdi-stream/unit-test-data
ADD ./config /cvdi-stream/config

# Do the build.
# do the build
RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make

# === RUNTIME IMAGE ===
FROM runtime-deps
USER root

WORKDIR /cvdi-stream

# copy the built files from the builder
COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/
COPY --from=builder /cvdi-stream /cvdi-stream

# Add test data. This changes frequently so keep it low in the file.
ADD ./docker-test /cvdi-stream/docker-test

Expand Down
54 changes: 35 additions & 19 deletions Dockerfile-nsv
Original file line number Diff line number Diff line change
@@ -1,4 +1,20 @@
FROM ubuntu:18.04
# === RUNTIME DEPENDENCIES IMAGE ===
FROM alpine:3.12 as runtime-deps
USER root

WORKDIR /cvdi-stream

# update the package manager
RUN apk update

# add runtime dependencies
RUN apk add --upgrade --no-cache \
bash \
librdkafka \
librdkafka-dev

# === BUILDER IMAGE ===
FROM runtime-deps as builder
USER root
ARG PPM_CONFIG_FILE
ARG PPM_MAP_FILE
Expand All @@ -7,21 +23,11 @@ WORKDIR /cvdi-stream

ENV DEBIAN_FRONTEND=noninteractive

# Add build tools.
RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100

# Install cmake.
RUN apt install -y libprotobuf-dev protobuf-compiler
RUN apt install -y cmake

# Install librdkafka.
RUN apt-get install -y sudo
RUN wget -qO - https://packages.confluent.io/deb/7.3/archive.key | sudo apt-key add -
RUN add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/7.3 stable main"
RUN add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main"
RUN apt update
RUN apt-get install -y libsasl2-modules libsasl2-modules-gssapi-mit libsasl2-dev libssl-dev
RUN apt install -y librdkafka-dev
# add build dependencies
RUN apk add --upgrade --no-cache --virtual .build-deps \
cmake \
g++ \
make

# add the source and build files
ADD CMakeLists.txt /cvdi-stream
Expand All @@ -34,11 +40,21 @@ ADD config/${PPM_CONFIG_FILE} /cvdi-stream/config/
ADD data/${PPM_MAP_FILE} /cvdi-stream/config/
ADD ./config /cvdi-stream/config

# Do the build.
# do the build
RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make

# Add test data. This changes frequently so keep it low in the file.
# === RUNTIME IMAGE ===
FROM runtime-deps
USER root

WORKDIR /cvdi-stream

# copy the built files from the builder
COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/
COPY --from=builder /cvdi-stream /cvdi-stream

# add test data (this changes frequently so keep it low in the file)
ADD ./docker-test /cvdi-stream/docker-test

# Run the tool.
# run the tool
CMD ["/cvdi-stream/docker-test/ppm-nsv.sh"]
50 changes: 33 additions & 17 deletions Dockerfile.standalone
Original file line number Diff line number Diff line change
@@ -1,25 +1,31 @@
FROM ubuntu:18.04
# === RUNTIME DEPENDENCIES IMAGE ===
FROM alpine:3.12 as runtime-deps
USER root

WORKDIR /cvdi-stream

ENV DEBIAN_FRONTEND=noninteractive
# update the package manager
RUN apk update

# add runtime dependencies
RUN apk add --upgrade --no-cache \
bash \
librdkafka \
librdkafka-dev

# === BUILDER IMAGE ===
FROM runtime-deps as builder
USER root

# Add build tools.
RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
WORKDIR /cvdi-stream

# Install cmake.
RUN apt install -y libprotobuf-dev protobuf-compiler
RUN apt install -y cmake
ENV DEBIAN_FRONTEND=noninteractive

# Install librdkafka.
RUN apt-get install -y sudo
RUN wget -qO - https://packages.confluent.io/deb/7.3/archive.key | sudo apt-key add -
RUN add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/7.3 stable main"
RUN add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main"
RUN apt update
RUN apt-get install -y libsasl2-modules libsasl2-modules-gssapi-mit libsasl2-dev libssl-dev
RUN apt install -y librdkafka-dev
# add build dependencies
RUN apk add --upgrade --no-cache --virtual .build-deps \
cmake \
g++ \
make

# add the source and build files
ADD CMakeLists.txt /cvdi-stream
Expand All @@ -30,8 +36,18 @@ ADD ./kafka-test /cvdi-stream/kafka-test
ADD ./unit-test-data /cvdi-stream/unit-test-data
ADD ./config /cvdi-stream/config

# Do the build.
# do the build
RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make

# Add test data. This changes frequently so keep it low in the file.
# === RUNTIME IMAGE ===
FROM runtime-deps
USER root

WORKDIR /cvdi-stream

# copy the built files from the builder
COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/
COPY --from=builder /cvdi-stream /cvdi-stream

# add test data (this changes frequently so keep it low in the file)
ADD ./docker-test /cvdi-stream/docker-test
42 changes: 31 additions & 11 deletions Dockerfile.testing
Original file line number Diff line number Diff line change
@@ -1,22 +1,32 @@
FROM ubuntu:18.04
# === RUNTIME DEPENDENCIES IMAGE ===
FROM alpine:3.12 as runtime-deps
USER root

WORKDIR /cvdi-stream

ENV DEBIAN_FRONTEND=noninteractive
# update the package manager
RUN apk update

# Add build tools.
RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
# add runtime dependencies
RUN apk add --upgrade --no-cache \
bash \
python3 \
librdkafka \
librdkafka-dev

# Install cmake.
RUN apt install -y libprotobuf-dev protobuf-compiler
RUN apt install -y cmake
# === BUILDER IMAGE ===
FROM runtime-deps as builder
USER root

# Install librdkafka.
RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev
WORKDIR /cvdi-stream

ENV DEBIAN_FRONTEND=noninteractive

# install python for testing
RUN apt-get install -y python3
# add build dependencies
RUN apk add --upgrade --no-cache --virtual .build-deps \
cmake \
g++ \
make

# add the source and build files
ADD CMakeLists.txt /cvdi-stream
Expand All @@ -30,6 +40,16 @@ ADD ./config /cvdi-stream/config
# Do the build.
RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make

# === RUNTIME IMAGE ===
FROM runtime-deps
USER root

WORKDIR /cvdi-stream

# copy the built files from the builder
COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/
COPY --from=builder /cvdi-stream /cvdi-stream

# Add test data. This changes frequently so keep it low in the file.
ADD ./docker-test /cvdi-stream/docker-test

Expand Down
16 changes: 11 additions & 5 deletions do_kafka_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@ CYAN='\033[0;36m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

CURRENT_DIR_NAME=${PWD##*/}
KAFKA_CONTAINER_NAME=$CURRENT_DIR_NAME-kafka-1

MAP_FILE=data/I_80.edges
BSM_DATA_FILE=data/I_80_test.json
TIM_DATA_FILE=data/I_80_test_TIMS.json
Expand All @@ -22,15 +21,21 @@ PPM_IMAGE_TAG=do-kafka-test-ppm-image
PPM_IMAGE_NAME=jpo-cvdp_ppm

setup() {
if [ -z $DOCKER_HOST_IP ]; then
echo "DOCKER_HOST_IP is not set. Exiting."
if [ -z $DOCKER_HOST_IP ]
then
export DOCKER_HOST_IP=$(ifconfig | grep -A 1 'inet ' | grep -v 'inet6\|127.0.0.1' | awk '{print $2}' | grep -E '^172\.1[6-9]\.|^172\.2[0-9]\.|^172\.3[0-1]\.|^192\.168\.' | head -n 1)
fi
if [ -z $DOCKER_HOST_IP ]
then
echo "DOCKER_HOST_IP is not set and could not be determined. Exiting."
exit 1
fi


# print setup info
echo "=== Setup Info ==="
echo "DOCKER_HOST_IP: $DOCKER_HOST_IP"
echo "KAFKA_CONTAINER_NAME: $KAFKA_CONTAINER_NAME"
echo "KAFKA_CONTAINER_NAME is resolved dynamically"
echo "MAP_FILE: $MAP_FILE"
echo "BSM_DATA_FILE: $BSM_DATA_FILE"
echo "TIM_DATA_FILE: $TIM_DATA_FILE"
Expand All @@ -45,6 +50,7 @@ setup() {
waitForKafkaToCreateTopics() {
maxAttempts=100
attempts=0
KAFKA_CONTAINER_NAME=$(docker ps --format '{{.Names}}' | grep kafka)
while true; do
attempts=$((attempts+1))
if [ $(docker ps | grep $KAFKA_CONTAINER_NAME | wc -l) == "0" ]; then
Expand Down
15 changes: 15 additions & 0 deletions docs/Release_notes.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,21 @@
Jpo-cvdp Release Notes
----------------------------

Version 1.3.0, released February 2024

### **Summary**
The changes for the jpo-cvdp 1.3.0 release involve the optimization of dockerfiles, addition of dockerhub image documentation & some QoL changes to the `do_kafka_test.sh` script.

Enhancements in this release:
- CDOT PR 32: Optimized dockerfiles for smaller & faster deployments by switching over to using alpine as the base image.
- CDOT PR 37: Added dockerhub image documentation
- CDOT PR 38: Updated `do_kafka_test.sh` script to set DOCKER_HOST_IP automatically
- CDOT PR 39: Updated `do_kafka_test.sh` script to resolve the KAFKA_CONTAINER_NAME env var dynamically

Known Issues:
- No known issues at this time.


Version 1.2.0, released November 2023
----------------------------------------

Expand Down
Loading

0 comments on commit c22b9d3

Please sign in to comment.