Merge pull request #26 from usdot-jpo-ode/release_1.1.0

Merge release branch into master branch

.github/workflows/docker.yml

@@ -0,0 +1,20 @@
+name: Docker build
+
+on:
+  push:
+    branches-ignore:
+      - "develop"
+      - "master"
+      - "release/*"
+  pull_request:
+
+jobs:
+  jpo-cvdp:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Build
+        uses: docker/build-push-action@v3

.github/workflows/dockerhub.yml

@@ -0,0 +1,26 @@
+name: "DockerHub Build and Push"
+
+on:
+  push:
+    branches:
+      - "develop"
+      - "master"
+      - "release/*"
+jobs:
+  dockerhub-Jpo-cvdp:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          tags: usdotjpoode/jpo-cvdp:${{ github.ref_name }}

CMakeLists.txt

@@ -1,20 +1,25 @@
+# Minimum required version of CMake
 cmake_minimum_required(VERSION 2.6)
 
+# Project name
 project(ppm)
 
-# need to set these prior to setting any targets.
+# Set C++ standard to 11 and make it required
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
+# Optimization flags
 set(CMAKE_CXX_FLAGS "-O3")
 set(CMAKE_C_FLAGS "-O3")
 
+# Set options for macOS
 if (${APPLE})
     set(CMAKE_CXX_EXTENSIONS OFF)
     set(MACPORTS_DIR "/opt")
 endif ()
 
-if(CMAKE_COMPILER_IS_GNUCXX)                                           # add coverage compiler option
+# Add coverage compiler option for GNU C++
+if(CMAKE_COMPILER_IS_GNUCXX)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage")
     set(CMAKE_CXX_OUTPUT_EXTENSION_REPLACE 1)
 endif()
@@ -32,53 +37,73 @@ include_directories("${CMAKE_CURRENT_SOURCE_DIR}/include/spdlog")
 include_directories("${CMAKE_CURRENT_SOURCE_DIR}/include/general-redaction")
 include_directories("/usr/local/include")
 
+# Add include directory for macOS
 if (${APPLE})
-    include_directories( "${MACPORTS_DIR}/local/include")
+    include_directories("${MACPORTS_DIR}/local/include")
     link_directories("${MACPORTS_DIR}/local/lib" "/usr/lib" "/usr/local/lib")
 endif ()
 
-#### BUILD TARGET FOR THE PPM ####
-
-set(PPM_SRC "src/general-redaction/redactionPropertiesManager.cpp"
-            "src/general-redaction/rapidjsonRedactor.cpp"
-            "src/bsm.cpp"
-            "src/bsmHandler.cpp"
-            "src/idRedactor.cpp"
-            "src/ppm.cpp"
-            "src/tool.cpp"
-            "src/velocityFilter.cpp"
-            "src/ppmLogger.cpp"
-             )
-
-add_executable(ppm ${PPM_SRC})
-target_link_libraries(ppm pthread CVLib rdkafka++)
-
-#### BUILD TARGET FOR THE PPM UNIT TESTS AND CODE COVERAGE ####
-
-set(PPM_TEST_SRC "src/tests.cpp")                                      # unit tests
-
+#### Build target for the PPM
+# List all the source files in project
+set(SOURCES
+    "src/general-redaction/redactionPropertiesManager.cpp"
+    "src/general-redaction/rapidjsonRedactor.cpp"
+    "src/bsm.cpp"
+    "src/bsmHandler.cpp"
+    "src/idRedactor.cpp"
+    "src/tool.cpp"
+    "src/velocityFilter.cpp"
+    "src/ppmLogger.cpp"
+)
+
+# Create a library target for the shared sources
+add_library(ppm-lib STATIC ${SOURCES})
+
+# Link the library target with the Kafka libraries
+target_link_libraries(ppm-lib PUBLIC 
+    rdkafka
+    rdkafka++ 
+)
+
+#### Create a target for the PPM executable
+add_executable(ppm "src/ppm.cpp")
+
+# Link the PPM executable with the PPM library target
+target_link_libraries(ppm PUBLIC ppm-lib CVLib)
+
+#### Create a target for the Kafka consumer executable
+add_executable(kafka_consumer "src/kafka_consumer.cpp")
+
+# Link the Kafka consumer executable with the PPM library target and the Kafka libraries
+target_link_libraries(kafka_consumer PUBLIC ppm-lib 
+    rdkafka
+    rdkafka++ 
+    CVLib
+)
+
+#### Build target for the PPM unit tests and code coverage
+set(PPM_TEST_SRC "src/tests.cpp")   # unit tests
+
+# Include the Catch header-only test framework
 set(CATCH_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/catch")
 add_library(Catch INTERFACE)
-target_include_directories(Catch INTERFACE ${CATCH_INCLUDE_DIR})       # catch is header only; tell where to find header.
+target_include_directories(Catch INTERFACE ${CATCH_INCLUDE_DIR})
 
-
-add_executable(ppm_tests ${PPM_TEST_SRC} ${PPM_SRC})                   # need everything to build tests.
+# Build the tests executable
+add_executable(ppm_tests ${PPM_TEST_SRC} ${SOURCES})
 target_link_libraries(ppm_tests pthread CVLib rdkafka++ Catch)
-target_compile_definitions(ppm_tests PRIVATE _PPM_TESTS)               # flag to exclude the tool's main.
-
-#### BUILD TARGET FOR THE KAFKA TEST TOOL ####
+target_compile_definitions(ppm_tests PRIVATE _PPM_TESTS)
 
+#### Build target for the Kafka test tool
 add_subdirectory(kafka-test)
 
-# Copy the data to the build. TODO make this part of the test or data target.
+# Copy the data and config directories to the build directory
 set(BSM_DATA_DIR $<TARGET_FILE_DIR:ppm>/unit-test-data)
 set(BSM_CONFIG_DIR $<TARGET_FILE_DIR:ppm>/config)
 
-# Make the base data directory.
+# Make the base data directory and copy the data files
 add_custom_command(TARGET ppm PRE_BUILD COMMAND ${CMAKE_COMMAND} 
                    -E make_directory ${BSM_DATA_DIR})
-# Copy the data files.
-add_custom_command(TARGET ppm PRE_BUILD COMMAND echo "Copying the data directory")
 add_custom_command(TARGET ppm PRE_BUILD COMMAND ${CMAKE_COMMAND} 
                    -E copy_directory ${PROJECT_SOURCE_DIR}/unit-test-data
                    ${BSM_DATA_DIR})
@@ -91,9 +116,3 @@ add_custom_command(TARGET ppm POST_BUILD COMMAND echo "Copying the config direct
 add_custom_command(TARGET ppm POST_BUILD COMMAND ${CMAKE_COMMAND} 
                    -E copy_directory ${PROJECT_SOURCE_DIR}/config
                    ${BSM_CONFIG_DIR})
-
-# option(BUILD_TESTS "Determines whether to build tests." ON)
-# if(BUILD_TESTS)
-#     enable_testing()
-#     add_test(NAME mytest1 COMMAND ppm_tests)
-# endif()

Dockerfile

@@ -4,14 +4,20 @@ USER root
 WORKDIR /cvdi-stream
 
 # Add build tools.
-RUN apt-get update && apt-get install -y g++
+RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
 
 # Install cmake.
 RUN apt install -y libprotobuf-dev protobuf-compiler
 RUN apt install -y cmake
 
 # Install librdkafka.
-RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev
+RUN apt-get install -y sudo
+RUN wget -qO - https://packages.confluent.io/deb/7.3/archive.key | sudo apt-key add -
+RUN add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/7.3 stable main"
+RUN add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main"
+RUN apt update
+RUN apt-get install -y libsasl2-modules libsasl2-modules-gssapi-mit libsasl2-dev libssl-dev 
+RUN apt install -y librdkafka-dev
 
 # add the source and build files
 ADD CMakeLists.txt /cvdi-stream

Dockerfile-nsv

@@ -6,14 +6,20 @@ ARG PPM_MAP_FILE
 WORKDIR /cvdi-stream
 
 # Add build tools.
-RUN apt-get update && apt-get install -y g++
+RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
 
 # Install cmake.
 RUN apt install -y libprotobuf-dev protobuf-compiler
 RUN apt install -y cmake
 
 # Install librdkafka.
-RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev
+RUN apt-get install -y sudo
+RUN wget -qO - https://packages.confluent.io/deb/7.3/archive.key | sudo apt-key add -
+RUN add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/7.3 stable main"
+RUN add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main"
+RUN apt update
+RUN apt-get install -y libsasl2-modules libsasl2-modules-gssapi-mit libsasl2-dev libssl-dev 
+RUN apt install -y librdkafka-dev
 
 # add the source and build files
 ADD CMakeLists.txt /cvdi-stream

Dockerfile.standalone

@@ -4,14 +4,20 @@ USER root
 WORKDIR /cvdi-stream
 
 # Add build tools.
-RUN apt-get update && apt-get install -y g++
+RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
 
 # Install cmake.
 RUN apt install -y libprotobuf-dev protobuf-compiler
 RUN apt install -y cmake
 
 # Install librdkafka.
-RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev
+RUN apt-get install -y sudo
+RUN wget -qO - https://packages.confluent.io/deb/7.3/archive.key | sudo apt-key add -
+RUN add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/7.3 stable main"
+RUN add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main"
+RUN apt update
+RUN apt-get install -y libsasl2-modules libsasl2-modules-gssapi-mit libsasl2-dev libssl-dev 
+RUN apt install -y librdkafka-dev
 
 # add the source and build files
 ADD CMakeLists.txt /cvdi-stream

README.md

@@ -79,9 +79,9 @@ The documentation is in HTML and is written to the `<install root>/jpo-cvdp/docs
 browser.
 
 ## Class Usage Diagram
-![class usage](./docs/diagrams/class-usage/PPM%20Class%20Usage%20With%20Files.drawio.png)
+![class usage](./docs/diagrams/class-usage/PPM%20Class%20Usage.drawio.png)
 
-This diagram shows the usage relationship between the classes of the project. Classes that are in the same file share the same white box. A class that uses another class will have a black arrow pointing to the latter. The PPM class extends the Tool class, and this is shown with a white arrow.
+This diagram displays how the different classes in the project are used. If one class uses another class, there will be a black arrow pointing to the class it uses. The Tool class is extended by the PPM class, which is represented by a white arrow.
 
 # Development and Collaboration Tools
 
@@ -136,8 +136,8 @@ Rather than using a local kafka instance, this project can utilize an instance o
 ## Environment variables
 ### Purpose & Usage
 - The DOCKER_HOST_IP environment variable is used to communicate with the bootstrap server that the instance of Kafka is running on.
-- The KAFKA_TYPE environment variable specifies what type of kafka connection will be attempted and is used to check if Confluent should be utilized.
-- The CONFLUENT_KEY and CONFLUENT_SECRET environment variables are used to authenticate with the bootstrap server.
+- The KAFKA_TYPE environment variable specifies what type of kafka connection will be attempted and is used to check if Confluent should be utilized. If this is not set to "CONFLUENT", the PPM will attempt to connect to a local kafka instance.
+- The CONFLUENT_KEY and CONFLUENT_SECRET environment variables are used to authenticate with the bootstrap server. These are the API key and secret that are generated when a new API key is created in Confluent Cloud. These are only used if the KAFKA_TYPE environment variable is set to "CONFLUENT".
 
 ### Values
 - DOCKER_HOST_IP must be set to the bootstrap server address (excluding the port)
@@ -182,6 +182,9 @@ When running the project in the provided dev container, the REDACTION_PROPERTIES
 #### RPM Debug
 If the RPM_DEBUG environment variable is set to true, debug messages will be logged to a file by the RedactionPropertiesManager class. This will allow developers to see whether the environment variable is set, whether the file was found and whether a non-zero number of redaction fields were loaded in.
 
+## Build & Exec Script
+The [`build_and_exec.sh`](./build_and_exec.sh) script can be used to build a tagged image of the PPM, run the container & enter it with an interactive shell. This script can be used to test the PPM in a standalone environment.
+
 ## Some Notes
 - The tests for this project can be run after compilation by running the "ppm_tests" executable.
 - When manually compiling with WSL, librdkafka will sometimes not be recognized. This can be resolved by utilizing the provided dev environment.

build_and_exec.sh

@@ -0,0 +1,7 @@
+# format of tag: 01-01-2020t12.00pm (lowercase t, am, pm)
+tag=$(date +"%m-%d-%Yt%I.%M%p" | tr '[:upper:]' '[:lower:]')
+echo "Building ppm-test-$tag"
+docker build . -t ppm-test-$tag
+
+echo "Running ppm-test-$tag"
+docker run -it ppm-test-$tag /bin/bash

docker-compose-standalone.yml

@@ -22,6 +22,9 @@ services:
       dockerfile: Dockerfile
     environment:
       DOCKER_HOST_IP: ${DOCKER_HOST_IP}
+      KAFKA_TYPE: ${KAFKA_TYPE}
+      CONFLUENT_KEY: ${CONFLUENT_KEY}
+      CONFLUENT_SECRET: ${CONFLUENT_SECRET}
       PPM_CONFIG_FILE: ppmBsm.properties
       REDACTION_PROPERTIES_PATH: ${REDACTION_PROPERTIES_PATH}
       PPM_LOG_TO_FILE: ${PPM_LOG_TO_FILE}

docker-compose.yml

@@ -29,9 +29,13 @@ services:
     volumes:
       - ${DOCKER_SHARED_VOLUME}:/ppm_data
     environment:
+      DOCKER_HOST_IP: ${DOCKER_HOST_IP}
+      KAFKA_TYPE: ${KAFKA_TYPE}
+      CONFLUENT_KEY: ${CONFLUENT_KEY}
+      CONFLUENT_SECRET: ${CONFLUENT_SECRET}
+      PPM_CONFIG_FILE: ${PPM_CONFIG_FILE}
       REDACTION_PROPERTIES_PATH: ${REDACTION_PROPERTIES_PATH}
       PPM_LOG_TO_FILE: ${PPM_LOG_TO_FILE}
       PPM_LOG_TO_CONSOLE: ${PPM_LOG_TO_CONSOLE}
       RPM_DEBUG: ${RPM_DEBUG}
-      PPM_CONFIG_FILE: ${PPM_CONFIG_FILE}
       PPM_LOG_LEVEL: ${PPM_LOG_LEVEL}

docs/diagrams/class-usage/PPM Class Usage.drawio

@@ -0,0 +1 @@
+<mxfile host="Electron" modified="2023-04-24T17:22:41.114Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/17.4.2 Chrome/100.0.4896.60 Electron/18.0.1 Safari/537.36" etag="xGXOI1zfdyAYW8VsL3YE" version="17.4.2" type="device"><diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1">7VxRd6o4EP41nrP74B5JQPSxWm27t73rvd617VNPKqnSAqEQq95fv0FAIKHU2kJk7VPJkCD5vsnMZDK0Afv26sxD7vyKGNhqgJaxasDTBgBAaWnsTyBZh5K22g0FM880QpGSCMbmbxwJW5F0YRrYz3SkhFjUdLPCKXEcPKUZGfI8ssx2eyBW9lddNMOCYDxFlii9Ng06D6UdoCfyc2zO5vEvK+1ofjaKO0cz8efIIMuUCA4asO8RQsMre9XHVgBejMv1xfraunxqn/39w39G//a+/fo+aYYPG75nyHYKHnbo3o+mmj+50YbG5Z07GoG76/vhzVn86BdkLSK8Bs7CPkf+PJozXcdAemThGDh4WKsBe8u5SfHYRdPg7pKpDpPNqW2xlsIuH0zL6hOLeJuxsN8faEP2Yr0dZxK/FvYoXqV4jGZ2homNqbdmXaK7sKWGQyIthe2ItGXCuRITOU/xHfdDkZrNto9OoGQXEZrvQBYKyF4YP7GBppSBUi9sY9xiC9CSja0qYDvBFpmadD00LYprhq/S1g4MX03Atze+qheoTQCyqMK4XQWqVn/xAGeg0713bXPyT/vMhaQJBASxwVxV1CQenZMZcZA1SKS9LMZJn0tC3AjZR0zpOvK7aEFJFne8MulNMPwvNv+weZu6dbqKHr1prOOGw+YbjtLi5m36XjJs04rHvY9Mnyy8KS5Qw8gRU+TNMC0ANnpegGahanjYQtR8yQYGeTRvhp54HlqnOrjEdKifevIoEKSWsZI1kxrg/DHXH2pF3dlF+AKJvm1nsr8KQqkqmNK/lDrmq2D5qlRk+SSrUpPzuFDXClVJ6K9p5euSLtec7W7NJKmS+tmq9CHn05HLlv5/cD5Fcb5s56NnLYDaecP5dAr7Zy2GMFrlRmsaFzOFqEWjOM39BNOjKDK0WZKCfXp0s6sd+T2+O9Hh7Tlmi+zGVskLfdQPKYIoDiC2ZkTJGJHEpkgzI7m4goMwI02VCyS6xWaE719oRvZY/UWQZnen58gxrLrt/DkzWmnSKn9/IMWwpjy+/i6XfwDRoKLuaMa7B7HAARcnQKV4gfNxBde/nJ2FmJT+iVzTePSJU88MalPbIYOqV7rQ5fjxZKGDvRa60tg9ANh/oS/BuKsub4E21CYX183Tl7U5jg5XZK1zMQTPZo8UPgIP51NaBA6PYPf/ITUoJ5EkBmB81hGouynCuyPDNvc7+huRIX+qwQ0oyXMAwXXgFcWO4Qvqykw3zWqXTz3yhGO/4BAHc64iEiHLnDmsOWUaxSJO2AscgTlF1kl0wzYNY6PweW4ouwjK8ji8n9+GkSmP085xOLAsh6OI56ELHx8dLa+t1gpoyd12tgRWZNj1/W10vq5psoz0x5aIeOx6hEukCbkdR4VLJJ+W9hctCeRxogfKZkX/YoXd7WaDLFWTTUvnixbRzcOOWHtSLS1izvQIaVE417I9xpJFCxAL2QYrZLsWA9zxFzbu3wsUHXaii09pS89oA9FPXK2D+tYRYvpJTaa9Xu1R3uYOq4C5edLXhqNT7blFJ9bUfT5f3Tzm7COO0L5wVThV7rlzSRHz6EdISpPLhGx9szRf3BIYqGCL7c+RG9x5sPDqJPisooSjsCj2e7uiQVpplDHWv3de1McrY2kPyGR1sQST+Og9fdy0OWVivmHkETdQaexfIQfN6na+3AS7lJaXdfCUC7YY8RyjTeID0VZ1+4NcVg6oqkeXXRdcZCNkl+eAfL3ZtS44rtN7/dDmlcK+zzq0KSp9SpmEkWtfklkNzS14Ox1TWlyei+2XuQ146XLLJudToLLMbWEAlFb5Uc0+sOJzXJXuQQvjzxSqvwixagprFTDmVjjIKQHcJxb4WIkv58vLrREpyE1VsO3JfUtwODy/8S1Y7XlWZfIspua+oYcnFGWYSw2xBLO3K/ivu3KFT7IpFX7Wm4tvDQtbyuCFK3Gt8MQrl5UaBr4VsKLokmkRS1z+sJHpMNHUQr7/55ERBLnvZ5ScZZOXntuDH9ZM/lVNuHVP/uEPHPwH</diagram></mxfile>

include/general-redaction/redactionPropertiesManager.hpp

@@ -69,12 +69,6 @@ class RedactionPropertiesManager {
          */
         void addField(std::string fieldToAdd);
 
-        /**
-         * @brief Prints the fields. For debugging purposes.
-         * 
-         */
-        void printFields();
-
     private:
         bool debug;
         std::vector<std::string> fieldsToRedact;