Merge pull request #35 from usdot-jpo-ode/release_q3

Merge Release Q3 into master

.github/workflows/ci.yml

@@ -0,0 +1,104 @@
+name: jpo-cvdp
+on: 
+ push:
+
+jobs:
+  jpo-cvdp:
+   runs-on: ubuntu-latest
+   container:
+     image: ubuntu:jammy-20230126
+     env:
+       REDACTION_PROPERTIES_PATH: "/__w/jpo-cvdp/jpo-cvdp/config/fieldsToRedact.txt"  # This env vairable is used to run ppm test
+       BUILD_WRAPPER_OUT_DIR: "$GITHUB_WORKSPACE/bw-output" # This env variable is needed to run SonarSource/sonarcloud-github-c-cpp@v1
+     options: "--user root"
+   steps:
+     - name: checkout code
+       uses: actions/checkout@v2 # This action to checkout the code
+     - name: Install Deps
+       run: |
+         apt update
+         apt-get -y install sudo wget curl gnupg lsb-release gcovr unzip
+         sudo apt-get -y install software-properties-common
+         curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+         sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+         sudo curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+         chmod +x /usr/local/bin/docker-compose
+         sudo apt-get -y update
+         sudo apt-get -y install docker-ce   
+     - name: set up Cmake
+       uses: jwlawson/actions-setup-cmake@v1.13  # this action is used to setup and install Cmake with required versions
+       with:
+         cmake-version: '3.16'  
+     - name: install g++
+       run: |
+         sudo apt-get -y install build-essential
+         sudo apt -y install cmake g++ libprotobuf-dev protobuf-compiler
+         sudo apt -y install apt-transport-https ca-certificates curl software-properties-common
+         export CXX="g++"          
+     - name: install librdkafka # This is to install librdkafka package
+       run: |
+         git clone --depth 1 https://github.com/confluentinc/librdkafka.git librdkafka
+         cd librdkafka
+         cmake -H. -B_cmake_build
+         cmake --build _cmake_build
+         cmake --build _cmake_build --target install  
+     - name: Install sonar-scanner and build-wrapper
+       uses: SonarSource/sonarcloud-github-c-cpp@v1  # This Action Installs sonar cloud and build wrapper to run sonar scan analysis  
+     - name: Set up Docker Buildx
+       uses: docker/setup-buildx-action@v2
+     - name: Build
+       uses: docker/build-push-action@v3       
+     - name: Build and Generate test coverage
+       run: |
+         cd $GITHUB_WORKSPACE
+         chmod   755  -R /__w/jpo-cvdp/jpo-cvdp
+         cat /usr/include/asm-generic/signal.h
+         cat /usr/include/x86_64-linux-gnu/asm/signal.h
+         export LD_LIBRARY_PATH=/usr/local/lib 
+         build-wrapper-linux-x86-64 --out-dir $GITHUB_WORKSPACE/bw-output ./build.sh     
+         mkdir coverage
+         cd coverage 
+         gcov $GITHUB_WORKSPACE/cv-lib/src/*.cpp --object-directory /__w/jpo-cvdp/jpo-cvdp/build/cv-lib/CMakeFiles/CVLib.dir/src/
+         gcov $GITHUB_WORKSPACE/src/*.cpp --object-directory /__w/jpo-cvdp/jpo-cvdp/build/CMakeFiles/ppm_tests.dir/src/              
+     - name: Archive code coverage results
+       uses: actions/upload-artifact@v3 # This action is used to capture the test artifacts and exits if no files are found
+       with:
+         name: jpo-cvdp
+         path: /__w/jpo-cvdp/jpo-cvdp/coverage/
+         if-no-files-found: error              
+     - name: Archive buildwrapper output
+       uses: actions/upload-artifact@v3  # This action is used to capture the builwrapper output files used by sonarscan.
+       with: 
+         name: jpo-cvdp
+         path: /home/runner/work/jpo-cvdp/jpo-cvdp/bw-output       
+     - name: Setup SonarScanner
+       uses: warchant/setup-sonar-scanner@v4 # This action is used to setup sonar scanner with required versions.
+       with:
+         version: 4.8.0.2856
+     - name: Generate sonar properties file
+       run: |
+         cat <<EOF > /tmp/sonar-scanner.properties
+         sonar.host.url=https://sonarcloud.io
+         sonar.projectName=jpo-cvdp
+         sonar.projectVersion=1.0
+         sonar.projectKey=usdot-jpo-ode_jpo-cvdp
+         sonar.organization=usdot-jpo-ode
+         sonar.sources=.
+         sonar.exclusions=src/ppm.cpp
+         sonar.cfamily.build-wrapper-output=bw-output
+         sonar.cfamily.gcov.reportsPath=/__w/jpo-cvdp/jpo-cvdp/coverage/
+         sonar.sonar.projectBaseDir=/home/runner/work/jpo-cvdp/jpo-cvdp/
+         sonar.exclusions=**/*.java
+         sonar.coverage.exclusions=**/*.java
+         # Set Git as SCM sensor
+         sonar.scm.disabled=true
+         #sonar.scm.enabled=false
+         sonar.scm.provider=git
+         sonar.sourceEncoding=UTF-8
+         EOF
+     - name: Run SonarScanner
+       uses: usdot-fhwa-stol/actions/sonar-scanner@main # This action runs the analysis using generated sonar gcov files and publish reports to respective sonarcloud project.
+       with:
+         sonar-properties-path: /tmp/sonar-scanner.properties
+         sonar-token: ${{ secrets.SONAR_TOKEN }}
+         working-dir: $GITHUB_WORKSPACE                   

Dockerfile

@@ -3,6 +3,8 @@ USER root
 
 WORKDIR /cvdi-stream
 
+ENV DEBIAN_FRONTEND=noninteractive
+
 # Add build tools.
 RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
 
@@ -26,6 +28,7 @@ ADD ./cv-lib /cvdi-stream/cv-lib
 ADD ./include /cvdi-stream/include
 ADD ./kafka-test /cvdi-stream/kafka-test
 ADD ./unit-test-data /cvdi-stream/unit-test-data
+ADD ./config /cvdi-stream/config
 
 # Do the build.
 RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make
@@ -34,5 +37,5 @@ RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvd
 ADD ./docker-test /cvdi-stream/docker-test
 
 # Run the tool.
-RUN chmod 7777 /cvdi-stream/docker-test/ppm_no_map.sh
+RUN chmod 777 /cvdi-stream/docker-test/ppm_no_map.sh
 CMD ["/cvdi-stream/docker-test/ppm_no_map.sh"]

Dockerfile-nsv

@@ -5,6 +5,8 @@ ARG PPM_MAP_FILE
 
 WORKDIR /cvdi-stream
 
+ENV DEBIAN_FRONTEND=noninteractive
+
 # Add build tools.
 RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
 
@@ -30,6 +32,7 @@ ADD ./kafka-test /cvdi-stream/kafka-test
 ADD ./unit-test-data /cvdi-stream/unit-test-data
 ADD config/${PPM_CONFIG_FILE} /cvdi-stream/config/
 ADD data/${PPM_MAP_FILE} /cvdi-stream/config/
+ADD ./config /cvdi-stream/config
 
 # Do the build.
 RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make

Dockerfile.standalone

@@ -3,6 +3,8 @@ USER root
 
 WORKDIR /cvdi-stream
 
+ENV DEBIAN_FRONTEND=noninteractive
+
 # Add build tools.
 RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
 
@@ -26,6 +28,7 @@ ADD ./cv-lib /cvdi-stream/cv-lib
 ADD ./include /cvdi-stream/include
 ADD ./kafka-test /cvdi-stream/kafka-test
 ADD ./unit-test-data /cvdi-stream/unit-test-data
+ADD ./config /cvdi-stream/config
 
 # Do the build.
 RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make

Dockerfile.testing

@@ -0,0 +1,38 @@
+FROM ubuntu:18.04
+USER root
+
+WORKDIR /cvdi-stream
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Add build tools.
+RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
+
+# Install cmake.
+RUN apt install -y libprotobuf-dev protobuf-compiler
+RUN apt install -y cmake
+
+# Install librdkafka.
+RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev
+
+# install python for testing
+RUN apt-get install -y python3
+
+# add the source and build files
+ADD CMakeLists.txt /cvdi-stream
+ADD ./src /cvdi-stream/src
+ADD ./cv-lib /cvdi-stream/cv-lib
+ADD ./include /cvdi-stream/include
+ADD ./kafka-test /cvdi-stream/kafka-test
+ADD ./unit-test-data /cvdi-stream/unit-test-data
+ADD ./config /cvdi-stream/config
+
+# Do the build.
+RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make
+
+# Add test data. This changes frequently so keep it low in the file.
+ADD ./docker-test /cvdi-stream/docker-test
+
+# Run the tool.
+RUN chmod 777 /cvdi-stream/docker-test/ppm_no_map.sh
+CMD ["/cvdi-stream/docker-test/ppm_no_map.sh"]

README.md

@@ -185,6 +185,32 @@ If the RPM_DEBUG environment variable is set to true, debug messages will be log
 ## Build & Exec Script
 The [`build_and_exec.sh`](./build_and_exec.sh) script can be used to build a tagged image of the PPM, run the container & enter it with an interactive shell. This script can be used to test the PPM in a standalone environment.
 
+This script should be run outside of the dev container in an environment where Docker is available.
+
+It should be noted that this script needs to use the LF end-of-line sequence.
+
+## Kafka Test Script
+The [do_kafka_test.sh](./do_kafka_test.sh) script is designed to perform integration tests on a Kafka instance. To execute the tests, this script relies on the following scripts: standalone.sh, standalone_multi.sh, do_bsm_test.sh, and do_tim_test.sh.
+
+To ensure proper execution, it is recommended to run this script outside of the dev container where docker is available. This is because the script will spin up a standalone kafka instance and will not be able to access the docker daemon from within the dev container.
+
+It should be noted that this script and any dependent scripts need to use the LF end-of-line sequence. These include the following:
+- do_kafka_test.sh
+- standalone.sh
+- standalone_multi.sh
+- do_bsm_test.sh
+- do_tim_test.sh
+- test_in.py
+- test_out.py
+
+The DOCKER_HOST_IP environment variable must be set to the IP address of the host machine. This is required for the script to function properly. This can be set by using the following command:
+
+```
+export DOCKER_HOST_IP=$(ifconfig | zgrep -m 1 -oP '(?<=inet\s)\d+(\.\d+){3}')
+```
+
+WSL will sometimes hang while the script waits for kafka to create topics. The script should exit after a number of attempts, but if it does not, running `wsl --shutdown` in a windows command prompt and restarting the docker services is recommended.
+
 ## Some Notes
 - The tests for this project can be run after compilation by running the "ppm_tests" executable.
 - When manually compiling with WSL, librdkafka will sometimes not be recognized. This can be resolved by utilizing the provided dev environment.

build.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+#  Copyright (C) 2018-2020 LEIDOS.
+# 
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may not
+#  use this file except in compliance with the License. You may obtain a copy of
+#  the License at
+# 
+#  http://www.apache.org/licenses/LICENSE-2.0
+# 
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations under
+#  the License.
+
+# script executes all kafka_clients and scheduling service build and coverage steps so that they can be singularly
+# wrapped by the sonarcloud build-wrapper
+set -e
+
+
+# check if REDACTION_PROPERTIES_PATH is set
+if [ -z "$REDACTION_PROPERTIES_PATH" ] ; then
+    echo "REDACTION_PROPERTIES_PATH is not set"
+    exit 1
+fi
+
+mkdir build
+cd build
+cmake ..
+cmake --build .
+
+./ppm_tests
+ctest --output-on-failure