Merge pull request #22 from usdot-jpo-ode/release/1.0.0

Merge release 1.0.0 into master

.devcontainer/Dockerfile

@@ -0,0 +1,29 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.224.2/containers/cpp/.devcontainer/base.Dockerfile
+
+# [Choice] Debian / Ubuntu version (use Debian 11, Ubuntu 18.04/21.04 on local arm64/Apple Silicon): debian-11, debian-10, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04
+ARG VARIANT="bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT}
+
+# [Optional] Install CMake version different from what base image has already installed. 
+# CMake reinstall choices: none, 3.21.5, 3.22.2, or versions from https://cmake.org/download/
+ARG REINSTALL_CMAKE_VERSION_FROM_SOURCE="none"
+
+# Optionally install the cmake for vcpkg
+COPY ./reinstall-cmake.sh /tmp/
+RUN if [ "${REINSTALL_CMAKE_VERSION_FROM_SOURCE}" != "none" ]; then \
+        /tmp/reinstall-cmake.sh ${REINSTALL_CMAKE_VERSION_FROM_SOURCE}; \
+    fi \
+    && rm -f /tmp/reinstall-cmake.sh
+
+# [Optional] Uncomment this section to install additional vcpkg ports.
+# RUN su vscode -c "${VCPKG_ROOT}/vcpkg install <your-port-name-here>"
+
+# [Optional] Uncomment this section to install additional packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+
+RUN apt-get update
+RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev
+
+ENV REDACTION_PROPERTIES_PATH /workspaces/jpo-cvdp/config/fieldsToRedact.txt
+ENV RPM_DEBUG true

.devcontainer/devcontainer.json

@@ -0,0 +1,30 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.224.2/containers/cpp
+{
+	"name": "C++",
+	"build": {
+		"dockerfile": "Dockerfile",
+		// Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04
+		// Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon
+		"args": { "VARIANT": "debian-11" }
+	},
+	"runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"],
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": {},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-vscode.cpptools",
+		"ms-vscode.cmake-tools"
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "gcc -v",
+
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode"
+}

.devcontainer/reinstall-cmake.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+#-------------------------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information.
+#-------------------------------------------------------------------------------------------------------------
+#
+set -e
+
+CMAKE_VERSION=${1:-"none"}
+
+if [ "${CMAKE_VERSION}" = "none" ]; then
+    echo "No CMake version specified, skipping CMake reinstallation"
+    exit 0
+fi
+
+# Cleanup temporary directory and associated files when exiting the script.
+cleanup() {
+    EXIT_CODE=$?
+    set +e
+    if [[ -n "${TMP_DIR}" ]]; then
+        echo "Executing cleanup of tmp files"
+        rm -Rf "${TMP_DIR}"
+    fi
+    exit $EXIT_CODE
+}
+trap cleanup EXIT
+
+
+echo "Installing CMake..."
+apt-get -y purge --auto-remove cmake
+mkdir -p /opt/cmake
+
+architecture=$(dpkg --print-architecture)
+case "${architecture}" in
+    arm64)
+        ARCH=aarch64 ;;
+    amd64)
+        ARCH=x86_64 ;;
+    *)
+        echo "Unsupported architecture ${architecture}."
+        exit 1
+        ;;
+esac
+
+CMAKE_BINARY_NAME="cmake-${CMAKE_VERSION}-linux-${ARCH}.sh"
+CMAKE_CHECKSUM_NAME="cmake-${CMAKE_VERSION}-SHA-256.txt"
+TMP_DIR=$(mktemp -d -t cmake-XXXXXXXXXX)
+
+echo "${TMP_DIR}"
+cd "${TMP_DIR}"
+
+curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/${CMAKE_BINARY_NAME}" -O
+curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/${CMAKE_CHECKSUM_NAME}" -O
+
+sha256sum -c --ignore-missing "${CMAKE_CHECKSUM_NAME}"
+sh "${TMP_DIR}/${CMAKE_BINARY_NAME}" --prefix=/opt/cmake --skip-license
+
+ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake

.gitattributes

@@ -0,0 +1,6 @@
+# Set the default behavior, in case people don't have core.autocrlf set.
+* text=auto
+
+# Explicitly declare bash scripts to be set to LF
+*.sh text eol=lf
+*.txt text eol=lf

CMakeLists.txt

@@ -29,6 +29,7 @@ include_directories(${CVLIB_INCLUDE})
 include_directories("${CMAKE_CURRENT_SOURCE_DIR}/include")
 include_directories("${CMAKE_CURRENT_SOURCE_DIR}/include/rapidjson")
 include_directories("${CMAKE_CURRENT_SOURCE_DIR}/include/spdlog")
+include_directories("${CMAKE_CURRENT_SOURCE_DIR}/include/general-redaction")
 include_directories("/usr/local/include")
 
 if (${APPLE})
@@ -38,9 +39,16 @@ endif ()
 
 #### BUILD TARGET FOR THE PPM ####
 
-set(PPM_SRC "src/ppm.cpp" 
-            "src/tool.cpp" 
-            "src/bsmfilter.cpp" )
+set(PPM_SRC "src/general-redaction/redactionPropertiesManager.cpp"
+            "src/general-redaction/rapidjsonRedactor.cpp"
+            "src/bsm.cpp"
+            "src/bsmHandler.cpp"
+            "src/idRedactor.cpp"
+            "src/ppm.cpp"
+            "src/tool.cpp"
+            "src/velocityFilter.cpp"
+            "src/ppmLogger.cpp"
+             )
 
 add_executable(ppm ${PPM_SRC})
 target_link_libraries(ppm pthread CVLib rdkafka++)

Dockerfile

@@ -4,14 +4,14 @@ USER root
 WORKDIR /cvdi-stream
 
 # Add build tools.
-RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
+RUN apt-get update && apt-get install -y g++
 
 # Install cmake.
-RUN wget https://cmake.org/files/v3.7/cmake-3.7.2.tar.gz && tar -xvf cmake-3.7.2.tar.gz
-RUN cd cmake-3.7.2 && ./bootstrap && make && make install && cd /
+RUN apt install -y libprotobuf-dev protobuf-compiler
+RUN apt install -y cmake
 
 # Install librdkafka.
-RUN git clone https://github.com/edenhill/librdkafka.git && cd librdkafka && ln -s /usr/bin/python3 /usr/bin/python && ./configure && make && make install && cd /cvdi-stream
+RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev
 
 # add the source and build files
 ADD CMakeLists.txt /cvdi-stream
@@ -28,4 +28,5 @@ RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvd
 ADD ./docker-test /cvdi-stream/docker-test
 
 # Run the tool.
-CMD ["/cvdi-stream/docker-test/ppm.sh"]
+RUN chmod 7777 /cvdi-stream/docker-test/ppm_no_map.sh
+CMD ["/cvdi-stream/docker-test/ppm_no_map.sh"]

Dockerfile-nsv

@@ -6,14 +6,14 @@ ARG PPM_MAP_FILE
 WORKDIR /cvdi-stream
 
 # Add build tools.
-RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
+RUN apt-get update && apt-get install -y g++
 
 # Install cmake.
-RUN wget https://cmake.org/files/v3.7/cmake-3.7.2.tar.gz && tar -xvf cmake-3.7.2.tar.gz
-RUN cd cmake-3.7.2 && ./bootstrap && make && make install && cd /
+RUN apt install -y libprotobuf-dev protobuf-compiler
+RUN apt install -y cmake
 
 # Install librdkafka.
-RUN git clone https://github.com/edenhill/librdkafka.git && cd librdkafka && ln -s /usr/bin/python3 /usr/bin/python && ./configure && make && make install && cd /cvdi-stream
+RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev
 
 # add the source and build files
 ADD CMakeLists.txt /cvdi-stream

Dockerfile.standalone

@@ -4,14 +4,14 @@ USER root
 WORKDIR /cvdi-stream
 
 # Add build tools.
-RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
+RUN apt-get update && apt-get install -y g++
 
 # Install cmake.
-RUN wget https://cmake.org/files/v3.7/cmake-3.7.2.tar.gz && tar -xvf cmake-3.7.2.tar.gz
-RUN cd cmake-3.7.2 && ./bootstrap && make && make install && cd /
+RUN apt install -y libprotobuf-dev protobuf-compiler
+RUN apt install -y cmake
 
 # Install librdkafka.
-RUN git clone https://github.com/edenhill/librdkafka.git && cd librdkafka && ln -s /usr/bin/python3 /usr/bin/python && ./configure && make && make install && cd /cvdi-stream
+RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev
 
 # add the source and build files
 ADD CMakeLists.txt /cvdi-stream

Doxyfile

@@ -758,7 +758,7 @@ WARN_LOGFILE           =
 # spaces.
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = ./src ./include ./cv-lib/src ./cv-lib/include
+INPUT                  = ./src ./include ./cv-lib/src ./cv-lib/include ./src/general-redaction ./include/general-redaction
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

README.md

@@ -49,34 +49,10 @@ loitering locations can aid in learning the identity of the driver.
 5. [Configuration and Operation](docs/configuration.md)
 6. [Testing](docs/testing.md)
 7. [Development](docs/coding-standards.md)
+8. [Confluent Cloud Integration](#confluent-cloud-integration)
 
 ## Release Notes
-
-### ODE Sprint 38
-
-- ODE-771: Fixed reported bug where the PPM exits when connections to Kafka brokers fail.
-
-### ODE Sprint 15
-
-- ODE-369/ORNL-15: Logging
-- Updated Identifier Redactor to include random assignment in lieu of fixed assignment.
-
-### ODE Sprint 14
-
-- ORNL-17: USDOT Playbook
-
-### ODE Sprint 13
-
-- ODE-290: Integration with the ODE.
-
-### ODE Sprint 12
-
-- ODE-77: Complete documentation
-
-### ODE Sprint 11
-
-- (Partial Complete) ODE-282 Implement a Module that Interfaces with the ODE.
-- (Partially Complete) ODE-77 Implement a PPM that uses a Geofence to Filter BSMs.
+The current version and release history of the Jpo-cvdp: [Jpo-cvdp Release Notes](<docs/Release_notes.md>)
 
 # Documentation
 
@@ -102,6 +78,11 @@ $ doxygen
 The documentation is in HTML and is written to the `<install root>/jpo-cvdp/docs/html` directory. Open `index.html` in a
 browser.
 
+## Class Usage Diagram
+![class usage](./docs/diagrams/class-usage/PPM%20Class%20Usage%20With%20Files.drawio.png)
+
+This diagram shows the usage relationship between the classes of the project. Classes that are in the same file share the same white box. A class that uses another class will have a black arrow pointing to the latter. The PPM class extends the Tool class, and this is shown with a white arrow.
+
 # Development and Collaboration Tools
 
 ## Source Repositories - GitHub
@@ -148,3 +129,88 @@ git config --global core.autocrlf false
 ```bash
 git clone https://github.com/usdot-jpo-ode/jpo-cvdp.git
 ```
+
+# Confluent Cloud Integration
+Rather than using a local kafka instance, this project can utilize an instance of kafka hosted by Confluent Cloud via SASL.
+
+## Environment variables
+### Purpose & Usage
+- The DOCKER_HOST_IP environment variable is used to communicate with the bootstrap server that the instance of Kafka is running on.
+- The KAFKA_TYPE environment variable specifies what type of kafka connection will be attempted and is used to check if Confluent should be utilized.
+- The CONFLUENT_KEY and CONFLUENT_SECRET environment variables are used to authenticate with the bootstrap server.
+
+### Values
+- DOCKER_HOST_IP must be set to the bootstrap server address (excluding the port)
+- KAFKA_TYPE must be set to "CONFLUENT"
+- CONFLUENT_KEY must be set to the API key being utilized for CC
+- CONFLUENT_SECRET must be set to the API secret being utilized for CC
+
+## CC Docker Compose File
+There is a provided docker-compose file (docker-compose-confluent-cloud.yml) that passes the above environment variables into the container that gets created. Further, this file doesn't spin up a local kafka instance since it is not required.
+
+## Note
+This has only been tested with Confluent Cloud but technically all SASL authenticated Kafka brokers can be reached using this method.
+
+# Testing/Troubleshooting
+## Unit Tests
+Unit tests can be built and executed using the build_and_run_unit_tests.sh file inside of the dev container for the project. More information about this can be found [here](./docs/testing.md#utilizing-the-build_and_run_unit_testssh-script).
+
+The unit tests are also built when the solution is compiled. For information on that, check out [this section](./docs/testing.md#unit-testing).
+
+## Standalone Cluster
+The docker-compose-standalone.yml file is meant for local testing/troubleshooting.
+
+To utilize this, pass the -f flag to the docker-compose command as follows:
+> docker-compose -f docker-compose-standalone.yml up
+
+Sometimes kafka will fail to start up properly. If this happens, spin down the containers and try again.
+
+### Data & Config Files
+Data and config files are expected to be in a location pointed to by the DOCKER_SHARED_VOLUME environment variable.
+
+At this time, the PPM assumes that this location is the /ppm_data directory. When run in a docker or k8s solution, an external drive/directory can be mounted to this directory.
+
+In a BSM configuration, the PPM requires the following files to be present in the /ppm_data directory:
+- *.edges
+- ppmBsm.properties
+
+#### fieldsToRedact.txt
+The path to this file is specified by the REDACTION_PROPERTIES_PATH environment variable. If this is not set, field redaction will not take place but the PPM will continue to function. If this is set and the file is not found, the same behavior will occur.
+
+When running the project in the provided dev container, the REDACTION_PROPERTIES_PATH environment variable should be set to the project-level fieldsToRedact.txt file for debugging/experimentation purposes. This is located in /workspaces/jpo-cvdp/config/fieldsToRedact.txt from the perspective of the dev container.
+
+#### RPM Debug
+If the RPM_DEBUG environment variable is set to true, debug messages will be logged to a file by the RedactionPropertiesManager class. This will allow developers to see whether the environment variable is set, whether the file was found and whether a non-zero number of redaction fields were loaded in.
+
+## Some Notes
+- The tests for this project can be run after compilation by running the "ppm_tests" executable.
+- When manually compiling with WSL, librdkafka will sometimes not be recognized. This can be resolved by utilizing the provided dev environment.
+
+# General Redaction
+General redaction refers to redaction functionality in the BSMHandler that utilizes the 'fieldsToRedact.txt' file to redact specified fields from BSM messages.
+
+## How to specify the fields to redact
+The fieldsToRedact.txt file is used by the BSMHandler and lists the paths to the fields to be redacted. It should be noted that this file needs to use the LF end-of-line sequence.
+
+### How are fields redacted?
+The paths in the fieldsToRedact.txt file area are added to a list and then used to search for the fields in the BSM message. If a member is found, the default behavior is to remove it with rapidjson's RemoveMember() function. It should be noted that by default, only leaf members are able to be removed. There are some exceptions to this which are listed in the [Overridden Redaction Behavior](#overridden-redaction-behavior) section.
+
+## Overridden Redaction Behavior
+Some values will be treated differently than others when redacted. For example, the 'coreData.angle' field will be set to 127 instead of being removed since it is a required field. The following table lists the overridden redaction behavior.
+
+| Field | Redaction Behavior |
+| --- | --- |
+| angle | Set to 127 |
+| transmission | Set to "UNAVAILABLE" |
+| wheelBrakes | Set first bit to 1 and all other bits to 0 |
+| weatherProbe | Remove object |
+| status | Remove object |
+| speedProfile | Remove object |
+| traction | Set to "unavailable" |
+| abs | Set to "unavailable" |
+| scs | Set to "unavailable" |
+| brakeBoost | Set to "unavailable" |
+| auxBrakes | Set to "unavailable" |
+
+### Bitstrings
+Since it would be incorrect for a bitstring to be missing bits, the PPM will remove the entire bitstring if any of its bits are redacted. This is done by removing the parent object. For example, if the 'partII.value.lights.lowBeamHeadlightsOn' field is redacted, the 'partII.value.lights' object will be removed.

build_and_run_unit_tests.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# check if REDACTION_PROPERTIES_PATH is set
+if [ -z "$REDACTION_PROPERTIES_PATH" ] ; then
+    echo "REDACTION_PROPERTIES_PATH is not set"
+    exit 1
+fi
+
+# remove build directory if it exists
+if [ -d build ] ; then 
+    rm -r build
+fi
+
+# build
+mkdir build 
+cd build
+cmake ..
+make
+
+# run unit tests
+./ppm_tests