diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 05435385..57e57d64 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -23,8 +23,8 @@ // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - // "postCreateCommand": "gcc -v", + "postCreateCommand": "sudo find /workspaces/ -type f -iname \"*.sh\" -exec chmod +x {} \\;" // Make all shell scripts executable // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode" + // "remoteUser": "vscode" } diff --git a/.gitattributes b/.gitattributes index 6888c150..add50b0e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,7 @@ # Set the default behavior, in case people don't have core.autocrlf set. -* text=auto +* text=auto eol=lf # Explicitly declare bash scripts to be set to LF *.sh text eol=lf -*.txt text eol=lf \ No newline at end of file +*.txt text eol=lf +*.py text eol=lf \ No newline at end of file diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml index 81f0790f..21f9b2fb 100644 --- a/.github/workflows/dockerhub.yml +++ b/.github/workflows/dockerhub.yml @@ -18,9 +18,14 @@ jobs: uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Replace Docker tag + id: set_tag + run: echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g')" >> $GITHUB_ENV + - name: Build uses: docker/build-push-action@v5 with: push: true - tags: usdotjpoode/jpo-cvdp:${{ github.ref_name }} + tags: usdotjpoode/jpo-cvdp:${{ env.TAG }} diff --git a/.gitignore b/.gitignore index f35d2df4..e793742a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ build docs/html /.env + +ppm_data/* \ No newline at end of file diff --git a/Dockerfile.standalone b/Dockerfile.standalone deleted file mode 100644 index bc786cd2..00000000 --- a/Dockerfile.standalone +++ /dev/null @@ -1,53 +0,0 @@ -# === RUNTIME DEPENDENCIES IMAGE === -FROM alpine:3.12 as runtime-deps -USER root - -WORKDIR /cvdi-stream - -# update the package manager -RUN apk update - -# add runtime dependencies -RUN apk add --upgrade --no-cache \ - bash \ - librdkafka \ - librdkafka-dev - -# === BUILDER IMAGE === -FROM runtime-deps as builder -USER root - -WORKDIR /cvdi-stream - -ENV DEBIAN_FRONTEND=noninteractive - -# add build dependencies -RUN apk add --upgrade --no-cache --virtual .build-deps \ - cmake \ - g++ \ - make - -# add the source and build files -ADD CMakeLists.txt /cvdi-stream -ADD ./src /cvdi-stream/src -ADD ./cv-lib /cvdi-stream/cv-lib -ADD ./include /cvdi-stream/include -ADD ./kafka-test /cvdi-stream/kafka-test -ADD ./unit-test-data /cvdi-stream/unit-test-data -ADD ./config /cvdi-stream/config - -# do the build -RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make - -# === RUNTIME IMAGE === -FROM runtime-deps -USER root - -WORKDIR /cvdi-stream - -# copy the built files from the builder -COPY --from=builder /cvdi-stream-build/ /cvdi-stream-build/ -COPY --from=builder /cvdi-stream /cvdi-stream - -# add test data (this changes frequently so keep it low in the file) -ADD ./docker-test /cvdi-stream/docker-test diff --git a/README.md b/README.md index bcd81ac1..116fa1bf 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,7 @@ -Master: [![Build Status](https://travis-ci.org/usdot-jpo-ode/jpo-cvdp.svg?branch=master)](https://travis-ci.org/usdot-jpo-ode/jpo-cvdp) [![Quality Gate](https://sonarqube.com/api/badges/gate?key=jpo-cvdp-key)](https://sonarqube.com/dashboard?id=jpo-cvdp-key) - # jpo-cvdp - The United States Department of Transportation Joint Program Office (JPO) -Connected Vehicle Data Privacy (CVDP) Project is developing a variety of methods -to enhance the privacy of individuals who generated connected vehicle data. +Connected Vehicle Data Privacy (CVDP) project is developing a variety of methods +to enhance the privacy of individuals who generate connected vehicle data. Connected vehicle technology uses in-vehicle wireless transceivers to broadcast and receive basic safety messages (BSMs) that include accurate spatiotemporal @@ -19,15 +16,14 @@ important.** Developing procedures that minimize the risk of associating trajectories with individuals is the objective of this project. # The Operational Data Environment (ODE) Privacy Protection Module (PPM) - -The PPM operates on streams of raw BSMs generated by the ODE. It determines +The PPM operates on streams of raw BSMs processed by the ODE. It determines whether individual BSMs should be retained or suppressed (deleted) based on the information in that BSM and auxiliary map information used to define a geofence. BSM geoposition (latitude and longitude) and speed are used to determine the -disposition of each BSM processed. The PPM also redacts other BSM fields. +disposition of each BSM processed. Additionally, the PPM redacts a configurable +set of fields from the BSMs that are retained to further protect privacy. ## PPM Limitations - Protecting against inference-based privacy attacks on spatiotemporal trajectories (i.e., sequences of BSMs from a single vehicle) in **general** is a challenging task. An example of an inference-based privacy attack is @@ -39,22 +35,30 @@ certain speed restrictions.** Do not assume this strategy will work in general. There are alternative strategies that must be employed to handle cases where loitering locations can aid in learning the identity of the driver. -## Table of Contents +## Supported Message Types +- Basic Safety Message (BSM) + +It should be noted that the PPM is not designed to handle other message types at this time. Future versions of the PPM may support additional message types. +## Table of Contents 1. [Release Notes](#release-notes) 2. [Documentation](#documentation) 3. [Development and Collaboration Tools](#development-and-collaboration-tools) -3. [Getting Started](#getting-started) -4. [Installation](docs/installation.md) -5. [Configuration and Operation](docs/configuration.md) -6. [Testing](docs/testing.md) -7. [Development](docs/coding-standards.md) -8. [Confluent Cloud Integration](#confluent-cloud-integration) +4. [Getting Started](#getting-started) +5. [Confluent Cloud Integration](#confluent-cloud-integration) +6. [Testing/Troubleshooting](#testingtroubleshooting) +7. [General Redaction](#general-redaction) + +### Additional Resources +1. [Installation](docs/installation.md) +2. [Configuration and Operation](docs/configuration.md) +3. [Testing](docs/testing.md) +4. [Development](docs/coding-standards.md) ## Release Notes The current version and release history of the Jpo-cvdp: [Jpo-cvdp Release Notes]() -# Documentation +## Documentation The following document will help practitioners build, test, deploy, and understand the PPM's functions: @@ -65,12 +69,13 @@ to the JPO Product Owner at DOT, FHWA, or JPO. To provide feedback, we recommend repository (https://github.com/usdot-jpo-ode/jpo-cvdp/issues). You will need a GitHub account to create an issue. If you don’t have an account, a dialog will be presented to you to create one at no cost. -## Code Documentation +### Code Documentation Code documentation can be generated using [Doxygen](https://www.doxygen.org) by following the commands below: ```bash -$ sudo apt install doxygen +$ sudo apt-get update +$ sudo apt-get install doxygen $ cd /jpo-cvdp $ doxygen ``` @@ -78,39 +83,32 @@ $ doxygen The documentation is in HTML and is written to the `/jpo-cvdp/docs/html` directory. Open `index.html` in a browser. -## Class Usage Diagram +### Class Usage Diagram ![class usage](./docs/diagrams/class-usage/PPM%20Class%20Usage.drawio.png) This diagram displays how the different classes in the project are used. If one class uses another class, there will be a black arrow pointing to the class it uses. The Tool class is extended by the PPM class, which is represented by a white arrow. -# Development and Collaboration Tools +## Development and Collaboration Tools -## Source Repositories - GitHub +### Source Repositories - GitHub - https://github.com/usdot-jpo-ode/jpo-cvdp - `git@github.com:usdot-jpo-ode/jpo-cvdp.git` -## Agile Project Management - Jira -https://usdotjpoode.atlassian.net/secure/Dashboard.jspa - -## Continuous Integration and Delivery +## Getting Started -The PPM is tested using [Travis Continuous Integration](https://travis-ci.org). - -# Getting Started - -## Prerequisites +### Prerequisites You will need Git to obtain the code and documents in this repository. Furthermore, we recommend using Docker to build the necessary containers to -build, test, and experiment with the PPM. The [Docker](#docker) instructions can be found in that section. +build, test, and experiment with the PPM. - [Git](https://git-scm.com/) - [Docker](https://www.docker.com) You can find more information in our [installation and setup](docs/installation.md) directions. -## Getting the Source Code +### Getting the Source Code See the installation and setup instructions unless you just want to examine the code. @@ -130,42 +128,40 @@ git config --global core.autocrlf false git clone https://github.com/usdot-jpo-ode/jpo-cvdp.git ``` -# Confluent Cloud Integration +## Confluent Cloud Integration Rather than using a local kafka instance, this project can utilize an instance of kafka hosted by Confluent Cloud via SASL. -## Environment variables -### Purpose & Usage +### Environment variables +#### Purpose & Usage - The DOCKER_HOST_IP environment variable is used to communicate with the bootstrap server that the instance of Kafka is running on. - The KAFKA_TYPE environment variable specifies what type of kafka connection will be attempted and is used to check if Confluent should be utilized. If this is not set to "CONFLUENT", the PPM will attempt to connect to a local kafka instance. - The CONFLUENT_KEY and CONFLUENT_SECRET environment variables are used to authenticate with the bootstrap server. These are the API key and secret that are generated when a new API key is created in Confluent Cloud. These are only used if the KAFKA_TYPE environment variable is set to "CONFLUENT". -### Values +#### Values - DOCKER_HOST_IP must be set to the bootstrap server address (excluding the port) - KAFKA_TYPE must be set to "CONFLUENT" - CONFLUENT_KEY must be set to the API key being utilized for CC - CONFLUENT_SECRET must be set to the API secret being utilized for CC -## CC Docker Compose File +### CC Docker Compose File There is a provided docker-compose file (docker-compose-confluent-cloud.yml) that passes the above environment variables into the container that gets created. Further, this file doesn't spin up a local kafka instance since it is not required. -## Note +### Note This has only been tested with Confluent Cloud but technically all SASL authenticated Kafka brokers can be reached using this method. -# Testing/Troubleshooting -## Unit Tests -Unit tests can be built and executed using the build_and_run_unit_tests.sh file inside of the dev container for the project. More information about this can be found [here](./docs/testing.md#utilizing-the-build_and_run_unit_testssh-script). - -The unit tests are also built when the solution is compiled. For information on that, check out [this section](./docs/testing.md#unit-testing). +## Testing/Troubleshooting +### Unit Tests +Unit tests can be built and executed using the build_and_run_unit_tests.sh file inside of the dev container for the project. Alternatively, they can be run inside of the deployed PPM container. More information about this can be found [here](./docs/testing.md#unit-testing). -## Standalone Cluster -The docker-compose-standalone.yml file is meant for local testing/troubleshooting. +### Standalone Cluster +The docker-compose.yml file is meant for local testing/troubleshooting. -To utilize this, pass the -f flag to the docker-compose command as follows: -> docker-compose -f docker-compose-standalone.yml up +To utilize this, run the following command in the root directory of the project: +> docker compose up Sometimes kafka will fail to start up properly. If this happens, spin down the containers and try again. -### Data & Config Files +#### Data & Config Files Data and config files are expected to be in a location pointed to by the DOCKER_SHARED_VOLUME environment variable. At this time, the PPM assumes that this location is the /ppm_data directory. When run in a docker or k8s solution, an external drive/directory can be mounted to this directory. @@ -174,32 +170,30 @@ In a BSM configuration, the PPM requires the following files to be present in th - *.edges - ppmBsm.properties -#### fieldsToRedact.txt +##### fieldsToRedact.txt The path to this file is specified by the REDACTION_PROPERTIES_PATH environment variable. If this is not set, field redaction will not take place but the PPM will continue to function. If this is set and the file is not found, the same behavior will occur. When running the project in the provided dev container, the REDACTION_PROPERTIES_PATH environment variable should be set to the project-level fieldsToRedact.txt file for debugging/experimentation purposes. This is located in /workspaces/jpo-cvdp/config/fieldsToRedact.txt from the perspective of the dev container. -#### RPM Debug +##### RPM Debug If the RPM_DEBUG environment variable is set to true, debug messages will be logged to a file by the RedactionPropertiesManager class. This will allow developers to see whether the environment variable is set, whether the file was found and whether a non-zero number of redaction fields were loaded in. -## Build & Exec Script +### Build & Exec Script The [`build_and_exec.sh`](./build_and_exec.sh) script can be used to build a tagged image of the PPM, run the container & enter it with an interactive shell. This script can be used to test the PPM in a standalone environment. This script should be run outside of the dev container in an environment where Docker is available. It should be noted that this script needs to use the LF end-of-line sequence. -## Kafka Test Script -The [do_kafka_test.sh](./do_kafka_test.sh) script is designed to perform integration tests on a Kafka instance. To execute the tests, this script relies on the following scripts: standalone.sh, standalone_multi.sh, do_bsm_test.sh, and do_tim_test.sh. +### Kafka Test Script +The [do_kafka_test.sh](./do_kafka_test.sh) script is designed to perform integration tests on a Kafka instance. To execute the tests, this script relies on the `standalone.sh` and `do_bsm_test.sh` scripts. To ensure proper execution, it is recommended to run this script outside of the dev container where docker is available. This is because the script will spin up a standalone kafka instance and will not be able to access the docker daemon from within the dev container. It should be noted that this script and any dependent scripts need to use the LF end-of-line sequence. These include the following: - do_kafka_test.sh - standalone.sh -- standalone_multi.sh - do_bsm_test.sh -- do_tim_test.sh - test_in.py - test_out.py @@ -211,20 +205,20 @@ export DOCKER_HOST_IP=$(ifconfig | zgrep -m 1 -oP '(?<=inet\s)\d+(\.\d+){3}') WSL will sometimes hang while the script waits for kafka to create topics. The script should exit after a number of attempts, but if it does not, running `wsl --shutdown` in a windows command prompt and restarting the docker services is recommended. -## Some Notes -- The tests for this project can be run after compilation by running the "ppm_tests" executable. -- When manually compiling with WSL, librdkafka will sometimes not be recognized. This can be resolved by utilizing the provided dev environment. +### Some Notes +- The tests for this project can be run after compilation by running the "ppm_tests" executable. An easy way to do this is to run the `build_and_exec.sh` script and then run the executable from within the container, which should be located in the /cvdi-stream-build directory. +- When manually compiling with WSL, librdkafka will sometimes not be recognized. This can be avoided by utilizing the provided dev environment. -# General Redaction +## General Redaction General redaction refers to redaction functionality in the BSMHandler that utilizes the 'fieldsToRedact.txt' file to redact specified fields from BSM messages. -## How to specify the fields to redact +### How to specify the fields to redact The fieldsToRedact.txt file is used by the BSMHandler and lists the paths to the fields to be redacted. It should be noted that this file needs to use the LF end-of-line sequence. -### How are fields redacted? +#### How are fields redacted? The paths in the fieldsToRedact.txt file area are added to a list and then used to search for the fields in the BSM message. If a member is found, the default behavior is to remove it with rapidjson's RemoveMember() function. It should be noted that by default, only leaf members are able to be removed. There are some exceptions to this which are listed in the [Overridden Redaction Behavior](#overridden-redaction-behavior) section. -## Overridden Redaction Behavior +### Overridden Redaction Behavior Some values will be treated differently than others when redacted. For example, the 'coreData.angle' field will be set to 127 instead of being removed since it is a required field. The following table lists the overridden redaction behavior. | Field | Redaction Behavior | @@ -241,5 +235,5 @@ Some values will be treated differently than others when redacted. For example, | brakeBoost | Set to "unavailable" | | auxBrakes | Set to "unavailable" | -### Bitstrings +#### Bitstrings Since it would be incorrect for a bitstring to be missing bits, the PPM will remove the entire bitstring if any of its bits are redacted. This is done by removing the parent object. For example, if the 'partII.value.lights.lowBeamHeadlightsOn' field is redacted, the 'partII.value.lights' object will be removed. diff --git a/config/ppmTim.properties b/config/ppmTim.properties deleted file mode 100644 index 3b2707ad..00000000 --- a/config/ppmTim.properties +++ /dev/null @@ -1,42 +0,0 @@ -# Configuration details for the velocity filter. -# min and max velocity values are in units m/s per the J2735 specification. -privacy.filter.velocity=OFF -privacy.filter.velocity.min=2.235 -privacy.filter.velocity.max=35.763 - -# Configuration details for privacy ID redaction. -privacy.redaction.id=OFF -privacy.redaction.id.value=FFFFFFFF -privacy.redaction.id.inclusions=OFF -privacy.redaction.id.included=BEA10000,BEA10001 - -# Configuration details for general redaction -privacy.redaction.general=OFF - -# Configuration details for geofencing. -privacy.filter.geofence=OFF -privacy.filter.geofence.mapfile=/ppm_data/I_80.edges -privacy.filter.geofence.sw.lat=40.997 -privacy.filter.geofence.sw.lon=-111.041 -privacy.filter.geofence.ne.lat=42.085 -privacy.filter.geofence.ne.lon=-104.047 - -# ODE / PPM Kafka topics. -privacy.topic.consumer=topic.OdeTimJson -privacy.topic.producer=topic.FilteredOdeTimJson - -group.id=PPM_TIM - -# max number of bytes per topic+partition to request from brokers -# defaults to 1 MiB, here we set it to 20 MiB -max.partition.fetch.bytes=20971520 - -# For testing purposes, use one partition. -# privacy.kafka.partition=0 - -# The host ip address for the Broker. -metadata.broker.list=your.kafka.broker.ip:9092 - -# specify the compression codec for all data generated: none, gzip, snappy, lz4 -compression.type=none - diff --git a/config/tim-test/I_80_vel_filter.properties b/config/tim-test/I_80_vel_filter.properties deleted file mode 100644 index c62ac9a7..00000000 --- a/config/tim-test/I_80_vel_filter.properties +++ /dev/null @@ -1,37 +0,0 @@ -# min and max velocity values to filter below and above (units m/s) -privacy.filter.velocity=ON -privacy.filter.velocity.min=2.235 -privacy.filter.velocity.max=35.763 - -# value to replace existing id for filtering. -privacy.redaction.id=OFF -privacy.redaction.id.value=FFFFFFFF -privacy.redaction.id.inclusions=ON -privacy.redaction.id.included=BEA10000,BEA10001 - -# geofence map fitting boundaries. -privacy.filter.geofence=OFF -privacy.filter.geofence.mapfile=/ppm_data/road_file.csv - -# geographic bounds that can further restrict the map information. -privacy.filter.geofence.sw.lat=40.970819 -privacy.filter.geofence.sw.lon=-111.090454 -privacy.filter.geofence.ne.lat=42.136908 -privacy.filter.geofence.ne.lon=-103.793068 - -# topics for the sanitization module to consume and produce. -privacy.topic.consumer=j2735BsmRawJson -privacy.topic.producer=j2735BsmFilteredJson - -group.id=0 - -privacy.kafka.partition=0 - -#metadata.broker.list=160.91.216.129:9092 -#metadata.broker.list=192.168.1.228:9092 -metadata.broker.list=172.17.0.1:9092 - -# specify the compression codec for all data generated: none, gzip, snappy, lz4 -compression.type=none - -debug=0 diff --git a/config/tim-test/c1.properties b/config/tim-test/c1.properties deleted file mode 100644 index 3fcb249c..00000000 --- a/config/tim-test/c1.properties +++ /dev/null @@ -1,41 +0,0 @@ -# min and max velocity values to filter below and above (units m/s) -privacy.filter.velocity=OFF -privacy.filter.velocity.min=2.235 -privacy.filter.velocity.max=35.763 - -# value to replace existing id for filtering. -privacy.redaction.id=OFF -privacy.redaction.id.value=FFFFFFFF -privacy.redaction.id.inclusions=ON -privacy.redaction.id.included=BEA10000,BEA10001 - -# geofence map fitting boundaries. -privacy.filter.geofence=OFF -privacy.filter.geofence.mapfile=/ppm_data/road_file.csv - -# geographic bounds that can further restrict the map information. -privacy.filter.geofence.sw.lat=40.970819 -privacy.filter.geofence.sw.lon=-111.090454 -privacy.filter.geofence.ne.lat=42.136908 -privacy.filter.geofence.ne.lon=-103.793068 -privacy.filter.geofence.extension=10.0 - -# topics for the sanitization module to consume and produce. -privacy.topic.consumer=topic.OdeTimJson -privacy.topic.producer=topic.FilteredOdeTimJson - -# Amount of time to wait when no message is available (milliseconds) -privacy.consumer.timeout=500 - -group.id=1 - -privacy.kafka.partition=0 - -#metadata.broker.list=160.91.216.129:9092 -#metadata.broker.list=192.168.1.228:9092 -metadata.broker.list=172.17.0.1:9092 - -# specify the compression codec for all data generated: none, gzip, snappy, lz4 -compression.type=none - -debug=0 diff --git a/config/tim-test/c2.properties b/config/tim-test/c2.properties deleted file mode 100644 index 642ddfa2..00000000 --- a/config/tim-test/c2.properties +++ /dev/null @@ -1,41 +0,0 @@ -# min and max velocity values to filter below and above (units m/s) -privacy.filter.velocity=ON -privacy.filter.velocity.min=2.235 -privacy.filter.velocity.max=35.763 - -# value to replace existing id for filtering. -privacy.redaction.id=OFF -privacy.redaction.id.value=FFFFFFFF -privacy.redaction.id.inclusions=ON -privacy.redaction.id.included=BEA10000,BEA10001 - -# geofence map fitting boundaries. -privacy.filter.geofence=OFF -privacy.filter.geofence.mapfile=/ppm_data/road_file.csv - -# geographic bounds that can further restrict the map information. -privacy.filter.geofence.sw.lat=40.970819 -privacy.filter.geofence.sw.lon=-111.090454 -privacy.filter.geofence.ne.lat=42.136908 -privacy.filter.geofence.ne.lon=-103.793068 -privacy.filter.geofence.extension=10.0 - -# topics for the sanitization module to consume and produce. -privacy.topic.consumer=topic.OdeTimJson -privacy.topic.producer=topic.FilteredOdeTimJson - -# Amount of time to wait when no message is available (milliseconds) -privacy.consumer.timeout=500 - -group.id=1 - -privacy.kafka.partition=0 - -#metadata.broker.list=160.91.216.129:9092 -#metadata.broker.list=192.168.1.228:9092 -metadata.broker.list=172.17.0.1:9092 - -# specify the compression codec for all data generated: none, gzip, snappy, lz4 -compression.type=none - -debug=0 diff --git a/config/tim-test/c3.properties b/config/tim-test/c3.properties deleted file mode 100644 index fc73e310..00000000 --- a/config/tim-test/c3.properties +++ /dev/null @@ -1,41 +0,0 @@ -# min and max velocity values to filter below and above (units m/s) -privacy.filter.velocity=OFF -privacy.filter.velocity.min=2.235 -privacy.filter.velocity.max=35.763 - -# value to replace existing id for filtering. -privacy.redaction.id=OFF -privacy.redaction.id.value=FFFFFFFF -privacy.redaction.id.inclusions=ON -privacy.redaction.id.included=BEA10000,BEA10001 - -# geofence map fitting boundaries. -privacy.filter.geofence=ON -privacy.filter.geofence.mapfile=/ppm_data/road_file.csv - -# geographic bounds that can further restrict the map information. -privacy.filter.geofence.sw.lat=40.970819 -privacy.filter.geofence.sw.lon=-111.090454 -privacy.filter.geofence.ne.lat=42.136908 -privacy.filter.geofence.ne.lon=-103.793068 -privacy.filter.geofence.extension=10.0 - -# topics for the sanitization module to consume and produce. -privacy.topic.consumer=topic.OdeTimJson -privacy.topic.producer=topic.FilteredOdeTimJson - -# Amount of time to wait when no message is available (milliseconds) -privacy.consumer.timeout=500 - -group.id=2 - -privacy.kafka.partition=0 - -#metadata.broker.list=160.91.216.129:9092 -#metadata.broker.list=192.168.1.228:9092 -metadata.broker.list=172.17.0.1:9092 - -# specify the compression codec for all data generated: none, gzip, snappy, lz4 -compression.type=none - -debug=0 diff --git a/data/I_80_test_TIMS.json b/data/I_80_test_TIMS.json deleted file mode 100644 index f6706b4a..00000000 --- a/data/I_80_test_TIMS.json +++ /dev/null @@ -1,10 +0,0 @@ -{"metadata": {"logFileName": "rxMsg_1507764909_2001_3A470_3A41af_3A1_3A226_3Aadff_3Afe05_3A2561.csv", "odeReceivedAt": "2017-12-06T23:15:59.596Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 1484, "heading": 193.6875, "latitude": 41.738136, "longitude": -106.587029, "speed": 7.02}, "rxSource": "RSU"}, "recordGeneratedAt": "2017-10-11T23:35:09.924Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "rxMsg", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 6, "bundleSize": 1, "recordId": 2, "serialNumber": 0, "streamId": "fef1b210-858b-4444-b0a9-72d4da9ad275"}, "validSignature": false}, "payload": {"data": {"MessageFrame": {"messageId": 31, "value": {"TravelerInformation": {"dataFrames": {"TravelerDataFrame": {"content": {"advisory": {"SEQUENCE": {"item": {"itis": 513}}}}, "duratonTime": 1440, "frameType": {"advisory": ""}, "msgId": {"roadSignID": {"mutcdCode": {"warning": ""}, "position": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "viewAngle": 1111111111111111}}, "priority": 5, "regions": {"GeographicalPath": {"anchor": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "closedPath": {"false": ""}, "description": {"path": {"offset": {"xy": {"nodes": {"NodeXY": [{"delta": {"node-LatLon": {"lat": 404735394, "lon": -1049701440}}}, {"delta": {"node-LatLon": {"lat": 404732984, "lon": -1049700032}}}, {"delta": {"node-LatLon": {"lat": 404730975, "lon": -1049699856}}}, {"delta": {"node-LatLon": {"lat": 404718186, "lon": -1049701616}}}, {"delta": {"node-LatLon": {"lat": 404716111, "lon": -1049702321}}}, {"delta": {"node-LatLon": {"lat": 404715307, "lon": -1049702937}}}]}}}}}, "direction": 1111111111111111, "directionality": {"both": ""}, "laneWidth": 800}}, "sspLocationRights": 1, "sspMsgRights1": 1, "sspMsgRights2": 1, "sspTimRights": 1, "startTime": 401108, "startYear": 2017}}, "msgCnt": 1, "packetID": "000000000018099C39"}}}}, "dataType": "TravelerInformation"}} -{"metadata": {"logFileName": "rxMsg_1507764909_2001_3A470_3A41af_3A1_3A226_3Aadff_3Afe05_3A2561.csv", "odeReceivedAt": "2017-12-06T23:15:59.596Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 1484, "heading": 193.6875, "latitude": 41.608656, "longitude": -109.226824, "speed": 7.12}, "rxSource": "RSU"}, "recordGeneratedAt": "2017-10-11T23:35:09.924Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "rxMsg", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 6, "bundleSize": 1, "recordId": 2, "serialNumber": 0, "streamId": "fef1b210-858b-4444-b0a9-72d4da9ad275"}, "validSignature": false}, "payload": {"data": {"MessageFrame": {"messageId": 31, "value": {"TravelerInformation": {"dataFrames": {"TravelerDataFrame": {"content": {"advisory": {"SEQUENCE": {"item": {"itis": 513}}}}, "duratonTime": 1440, "frameType": {"advisory": ""}, "msgId": {"roadSignID": {"mutcdCode": {"warning": ""}, "position": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "viewAngle": 1111111111111111}}, "priority": 5, "regions": {"GeographicalPath": {"anchor": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "closedPath": {"false": ""}, "description": {"path": {"offset": {"xy": {"nodes": {"NodeXY": [{"delta": {"node-LatLon": {"lat": 404735394, "lon": -1049701440}}}, {"delta": {"node-LatLon": {"lat": 404732984, "lon": -1049700032}}}, {"delta": {"node-LatLon": {"lat": 404730975, "lon": -1049699856}}}, {"delta": {"node-LatLon": {"lat": 404718186, "lon": -1049701616}}}, {"delta": {"node-LatLon": {"lat": 404716111, "lon": -1049702321}}}, {"delta": {"node-LatLon": {"lat": 404715307, "lon": -1049702937}}}]}}}}}, "direction": 1111111111111111, "directionality": {"both": ""}, "laneWidth": 800}}, "sspLocationRights": 1, "sspMsgRights1": 1, "sspMsgRights2": 1, "sspTimRights": 1, "startTime": 401108, "startYear": 2017}}, "msgCnt": 1, "packetID": "000000000018099C39"}}}}, "dataType": "TravelerInformation"}} -{"metadata": {"logFileName": "rxMsg_1507764909_2001_3A470_3A41af_3A1_3A226_3Aadff_3Afe05_3A2561.csv", "odeReceivedAt": "2017-12-06T23:15:59.596Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 1484, "heading": 193.6875, "latitude": 41.311097, "longitude": -110.512927, "speed": 7.16}, "rxSource": "RSU"}, "recordGeneratedAt": "2017-10-11T23:35:09.924Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "rxMsg", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 6, "bundleSize": 1, "recordId": 2, "serialNumber": 0, "streamId": "fef1b210-858b-4444-b0a9-72d4da9ad275"}, "validSignature": false}, "payload": {"data": {"MessageFrame": {"messageId": 31, "value": {"TravelerInformation": {"dataFrames": {"TravelerDataFrame": {"content": {"advisory": {"SEQUENCE": {"item": {"itis": 513}}}}, "duratonTime": 1440, "frameType": {"advisory": ""}, "msgId": {"roadSignID": {"mutcdCode": {"warning": ""}, "position": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "viewAngle": 1111111111111111}}, "priority": 5, "regions": {"GeographicalPath": {"anchor": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "closedPath": {"false": ""}, "description": {"path": {"offset": {"xy": {"nodes": {"NodeXY": [{"delta": {"node-LatLon": {"lat": 404735394, "lon": -1049701440}}}, {"delta": {"node-LatLon": {"lat": 404732984, "lon": -1049700032}}}, {"delta": {"node-LatLon": {"lat": 404730975, "lon": -1049699856}}}, {"delta": {"node-LatLon": {"lat": 404718186, "lon": -1049701616}}}, {"delta": {"node-LatLon": {"lat": 404716111, "lon": -1049702321}}}, {"delta": {"node-LatLon": {"lat": 404715307, "lon": -1049702937}}}]}}}}}, "direction": 1111111111111111, "directionality": {"both": ""}, "laneWidth": 800}}, "sspLocationRights": 1, "sspMsgRights1": 1, "sspMsgRights2": 1, "sspTimRights": 1, "startTime": 401108, "startYear": 2017}}, "msgCnt": 1, "packetID": "000000000018099C39"}}}}, "dataType": "TravelerInformation"}} -{"metadata": {"logFileName": "rxMsg_1507764909_2001_3A470_3A41af_3A1_3A226_3Aadff_3Afe05_3A2561.csv", "odeReceivedAt": "2017-12-06T23:15:59.596Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 1484, "heading": 193.6875, "latitude": 41.246647, "longitude": -111.027436, "speed": 7.44}, "rxSource": "RSU"}, "recordGeneratedAt": "2017-10-11T23:35:09.924Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "rxMsg", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 6, "bundleSize": 1, "recordId": 2, "serialNumber": 0, "streamId": "fef1b210-858b-4444-b0a9-72d4da9ad275"}, "validSignature": false}, "payload": {"data": {"MessageFrame": {"messageId": 31, "value": {"TravelerInformation": {"dataFrames": {"TravelerDataFrame": {"content": {"advisory": {"SEQUENCE": {"item": {"itis": 513}}}}, "duratonTime": 1440, "frameType": {"advisory": ""}, "msgId": {"roadSignID": {"mutcdCode": {"warning": ""}, "position": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "viewAngle": 1111111111111111}}, "priority": 5, "regions": {"GeographicalPath": {"anchor": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "closedPath": {"false": ""}, "description": {"path": {"offset": {"xy": {"nodes": {"NodeXY": [{"delta": {"node-LatLon": {"lat": 404735394, "lon": -1049701440}}}, {"delta": {"node-LatLon": {"lat": 404732984, "lon": -1049700032}}}, {"delta": {"node-LatLon": {"lat": 404730975, "lon": -1049699856}}}, {"delta": {"node-LatLon": {"lat": 404718186, "lon": -1049701616}}}, {"delta": {"node-LatLon": {"lat": 404716111, "lon": -1049702321}}}, {"delta": {"node-LatLon": {"lat": 404715307, "lon": -1049702937}}}]}}}}}, "direction": 1111111111111111, "directionality": {"both": ""}, "laneWidth": 800}}, "sspLocationRights": 1, "sspMsgRights1": 1, "sspMsgRights2": 1, "sspTimRights": 1, "startTime": 401108, "startYear": 2017}}, "msgCnt": 1, "packetID": "000000000018099C39"}}}}, "dataType": "TravelerInformation"}} -{"metadata": {"logFileName": "rxMsg_1507764909_2001_3A470_3A41af_3A1_3A226_3Aadff_3Afe05_3A2561.csv", "odeReceivedAt": "2017-12-06T23:15:59.596Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 1484, "heading": 193.6875, "latitude": 41.600371, "longitude": -106.22341, "speed": 7.44}, "rxSource": "RSU"}, "recordGeneratedAt": "2017-10-11T23:35:09.924Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "rxMsg", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 6, "bundleSize": 1, "recordId": 2, "serialNumber": 0, "streamId": "fef1b210-858b-4444-b0a9-72d4da9ad275"}, "validSignature": false}, "payload": {"data": {"MessageFrame": {"messageId": 31, "value": {"TravelerInformation": {"dataFrames": {"TravelerDataFrame": {"content": {"advisory": {"SEQUENCE": {"item": {"itis": 513}}}}, "duratonTime": 1440, "frameType": {"advisory": ""}, "msgId": {"roadSignID": {"mutcdCode": {"warning": ""}, "position": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "viewAngle": 1111111111111111}}, "priority": 5, "regions": {"GeographicalPath": {"anchor": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "closedPath": {"false": ""}, "description": {"path": {"offset": {"xy": {"nodes": {"NodeXY": [{"delta": {"node-LatLon": {"lat": 404735394, "lon": -1049701440}}}, {"delta": {"node-LatLon": {"lat": 404732984, "lon": -1049700032}}}, {"delta": {"node-LatLon": {"lat": 404730975, "lon": -1049699856}}}, {"delta": {"node-LatLon": {"lat": 404718186, "lon": -1049701616}}}, {"delta": {"node-LatLon": {"lat": 404716111, "lon": -1049702321}}}, {"delta": {"node-LatLon": {"lat": 404715307, "lon": -1049702937}}}]}}}}}, "direction": 1111111111111111, "directionality": {"both": ""}, "laneWidth": 800}}, "sspLocationRights": 1, "sspMsgRights1": 1, "sspMsgRights2": 1, "sspTimRights": 1, "startTime": 401108, "startYear": 2017}}, "msgCnt": 1, "packetID": "000000000018099C39"}}}}, "dataType": "TravelerInformation"}} -{"metadata": {"logFileName": "rxMsg_1507764909_2001_3A470_3A41af_3A1_3A226_3Aadff_3Afe05_3A2561.csv", "odeReceivedAt": "2017-12-06T23:15:59.596Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 1484, "heading": 193.6875, "latitude": 42.29789, "longitude": -83.72035, "speed": 1.78}, "rxSource": "RSU"}, "recordGeneratedAt": "2017-10-11T23:35:09.924Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "rxMsg", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 6, "bundleSize": 1, "recordId": 2, "serialNumber": 0, "streamId": "fef1b210-858b-4444-b0a9-72d4da9ad275"}, "validSignature": false}, "payload": {"data": {"MessageFrame": {"messageId": 31, "value": {"TravelerInformation": {"dataFrames": {"TravelerDataFrame": {"content": {"advisory": {"SEQUENCE": {"item": {"itis": 513}}}}, "duratonTime": 1440, "frameType": {"advisory": ""}, "msgId": {"roadSignID": {"mutcdCode": {"warning": ""}, "position": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "viewAngle": 1111111111111111}}, "priority": 5, "regions": {"GeographicalPath": {"anchor": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "closedPath": {"false": ""}, "description": {"path": {"offset": {"xy": {"nodes": {"NodeXY": [{"delta": {"node-LatLon": {"lat": 404735394, "lon": -1049701440}}}, {"delta": {"node-LatLon": {"lat": 404732984, "lon": -1049700032}}}, {"delta": {"node-LatLon": {"lat": 404730975, "lon": -1049699856}}}, {"delta": {"node-LatLon": {"lat": 404718186, "lon": -1049701616}}}, {"delta": {"node-LatLon": {"lat": 404716111, "lon": -1049702321}}}, {"delta": {"node-LatLon": {"lat": 404715307, "lon": -1049702937}}}]}}}}}, "direction": 1111111111111111, "directionality": {"both": ""}, "laneWidth": 800}}, "sspLocationRights": 1, "sspMsgRights1": 1, "sspMsgRights2": 1, "sspTimRights": 1, "startTime": 401108, "startYear": 2017}}, "msgCnt": 1, "packetID": "000000000018099C39"}}}}, "dataType": "TravelerInformation"}} -{"metadata": {"logFileName": "rxMsg_1507764909_2001_3A470_3A41af_3A1_3A226_3Aadff_3Afe05_3A2561.csv", "odeReceivedAt": "2017-12-06T23:15:59.596Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 1484, "heading": 193.6875, "latitude": 42.29789, "longitude": -83.72034, "speed": 0.7}, "rxSource": "RSU"}, "recordGeneratedAt": "2017-10-11T23:35:09.924Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "rxMsg", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 6, "bundleSize": 1, "recordId": 2, "serialNumber": 0, "streamId": "fef1b210-858b-4444-b0a9-72d4da9ad275"}, "validSignature": false}, "payload": {"data": {"MessageFrame": {"messageId": 31, "value": {"TravelerInformation": {"dataFrames": {"TravelerDataFrame": {"content": {"advisory": {"SEQUENCE": {"item": {"itis": 513}}}}, "duratonTime": 1440, "frameType": {"advisory": ""}, "msgId": {"roadSignID": {"mutcdCode": {"warning": ""}, "position": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "viewAngle": 1111111111111111}}, "priority": 5, "regions": {"GeographicalPath": {"anchor": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "closedPath": {"false": ""}, "description": {"path": {"offset": {"xy": {"nodes": {"NodeXY": [{"delta": {"node-LatLon": {"lat": 404735394, "lon": -1049701440}}}, {"delta": {"node-LatLon": {"lat": 404732984, "lon": -1049700032}}}, {"delta": {"node-LatLon": {"lat": 404730975, "lon": -1049699856}}}, {"delta": {"node-LatLon": {"lat": 404718186, "lon": -1049701616}}}, {"delta": {"node-LatLon": {"lat": 404716111, "lon": -1049702321}}}, {"delta": {"node-LatLon": {"lat": 404715307, "lon": -1049702937}}}]}}}}}, "direction": 1111111111111111, "directionality": {"both": ""}, "laneWidth": 800}}, "sspLocationRights": 1, "sspMsgRights1": 1, "sspMsgRights2": 1, "sspTimRights": 1, "startTime": 401108, "startYear": 2017}}, "msgCnt": 1, "packetID": "000000000018099C39"}}}}, "dataType": "TravelerInformation"}} -{"metadata": {"logFileName": "rxMsg_1507764909_2001_3A470_3A41af_3A1_3A226_3Aadff_3Afe05_3A2561.csv", "odeReceivedAt": "2017-12-06T23:15:59.596Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 1484, "heading": 193.6875, "latitude": 42.24576, "longitude": -83.62337, "speed": 6.86}, "rxSource": "RSU"}, "recordGeneratedAt": "2017-10-11T23:35:09.924Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "rxMsg", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 6, "bundleSize": 1, "recordId": 2, "serialNumber": 0, "streamId": "fef1b210-858b-4444-b0a9-72d4da9ad275"}, "validSignature": false}, "payload": {"data": {"MessageFrame": {"messageId": 31, "value": {"TravelerInformation": {"dataFrames": {"TravelerDataFrame": {"content": {"advisory": {"SEQUENCE": {"item": {"itis": 513}}}}, "duratonTime": 1440, "frameType": {"advisory": ""}, "msgId": {"roadSignID": {"mutcdCode": {"warning": ""}, "position": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "viewAngle": 1111111111111111}}, "priority": 5, "regions": {"GeographicalPath": {"anchor": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "closedPath": {"false": ""}, "description": {"path": {"offset": {"xy": {"nodes": {"NodeXY": [{"delta": {"node-LatLon": {"lat": 404735394, "lon": -1049701440}}}, {"delta": {"node-LatLon": {"lat": 404732984, "lon": -1049700032}}}, {"delta": {"node-LatLon": {"lat": 404730975, "lon": -1049699856}}}, {"delta": {"node-LatLon": {"lat": 404718186, "lon": -1049701616}}}, {"delta": {"node-LatLon": {"lat": 404716111, "lon": -1049702321}}}, {"delta": {"node-LatLon": {"lat": 404715307, "lon": -1049702937}}}]}}}}}, "direction": 1111111111111111, "directionality": {"both": ""}, "laneWidth": 800}}, "sspLocationRights": 1, "sspMsgRights1": 1, "sspMsgRights2": 1, "sspTimRights": 1, "startTime": 401108, "startYear": 2017}}, "msgCnt": 1, "packetID": "000000000018099C39"}}}}, "dataType": "TravelerInformation"}} -{"metadata": {"logFileName": "rxMsg_1507764909_2001_3A470_3A41af_3A1_3A226_3Aadff_3Afe05_3A2561.csv", "odeReceivedAt": "2017-12-06T23:15:59.596Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 1484, "heading": 193.6875, "latitude": 42.24576, "longitude": -83.62337, "speed": 6.84}, "rxSource": "RSU"}, "recordGeneratedAt": "2017-10-11T23:35:09.924Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "rxMsg", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 6, "bundleSize": 1, "recordId": 2, "serialNumber": 0, "streamId": "fef1b210-858b-4444-b0a9-72d4da9ad275"}, "validSignature": false}, "payload": {"data": {"MessageFrame": {"messageId": 31, "value": {"TravelerInformation": {"dataFrames": {"TravelerDataFrame": {"content": {"advisory": {"SEQUENCE": {"item": {"itis": 513}}}}, "duratonTime": 1440, "frameType": {"advisory": ""}, "msgId": {"roadSignID": {"mutcdCode": {"warning": ""}, "position": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "viewAngle": 1111111111111111}}, "priority": 5, "regions": {"GeographicalPath": {"anchor": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "closedPath": {"false": ""}, "description": {"path": {"offset": {"xy": {"nodes": {"NodeXY": [{"delta": {"node-LatLon": {"lat": 404735394, "lon": -1049701440}}}, {"delta": {"node-LatLon": {"lat": 404732984, "lon": -1049700032}}}, {"delta": {"node-LatLon": {"lat": 404730975, "lon": -1049699856}}}, {"delta": {"node-LatLon": {"lat": 404718186, "lon": -1049701616}}}, {"delta": {"node-LatLon": {"lat": 404716111, "lon": -1049702321}}}, {"delta": {"node-LatLon": {"lat": 404715307, "lon": -1049702937}}}]}}}}}, "direction": 1111111111111111, "directionality": {"both": ""}, "laneWidth": 800}}, "sspLocationRights": 1, "sspMsgRights1": 1, "sspMsgRights2": 1, "sspTimRights": 1, "startTime": 401108, "startYear": 2017}}, "msgCnt": 1, "packetID": "000000000018099C39"}}}}, "dataType": "TravelerInformation"}} -{"metadata": {"logFileName": "rxMsg_1507764909_2001_3A470_3A41af_3A1_3A226_3Aadff_3Afe05_3A2561.csv", "odeReceivedAt": "2017-12-06T23:15:59.596Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 1484, "heading": 193.6875, "latitude": 42.24576, "longitude": -83.62338, "speed": 6.74}, "rxSource": "RSU"}, "recordGeneratedAt": "2017-10-11T23:35:09.924Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "rxMsg", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 6, "bundleSize": 1, "recordId": 2, "serialNumber": 0, "streamId": "fef1b210-858b-4444-b0a9-72d4da9ad275"}, "validSignature": false}, "payload": {"data": {"MessageFrame": {"messageId": 31, "value": {"TravelerInformation": {"dataFrames": {"TravelerDataFrame": {"content": {"advisory": {"SEQUENCE": {"item": {"itis": 513}}}}, "duratonTime": 1440, "frameType": {"advisory": ""}, "msgId": {"roadSignID": {"mutcdCode": {"warning": ""}, "position": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "viewAngle": 1111111111111111}}, "priority": 5, "regions": {"GeographicalPath": {"anchor": {"elevation": 4096, "lat": 404725418, "long": -1049700560}, "closedPath": {"false": ""}, "description": {"path": {"offset": {"xy": {"nodes": {"NodeXY": [{"delta": {"node-LatLon": {"lat": 404735394, "lon": -1049701440}}}, {"delta": {"node-LatLon": {"lat": 404732984, "lon": -1049700032}}}, {"delta": {"node-LatLon": {"lat": 404730975, "lon": -1049699856}}}, {"delta": {"node-LatLon": {"lat": 404718186, "lon": -1049701616}}}, {"delta": {"node-LatLon": {"lat": 404716111, "lon": -1049702321}}}, {"delta": {"node-LatLon": {"lat": 404715307, "lon": -1049702937}}}]}}}}}, "direction": 1111111111111111, "directionality": {"both": ""}, "laneWidth": 800}}, "sspLocationRights": 1, "sspMsgRights1": 1, "sspMsgRights2": 1, "sspTimRights": 1, "startTime": 401108, "startYear": 2017}}, "msgCnt": 1, "packetID": "000000000018099C39"}}}}, "dataType": "TravelerInformation"}} diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 6d7c0637..420d2c69 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -15,7 +15,6 @@ NC='\033[0m' # No Color MAP_FILE=data/I_80.edges BSM_DATA_FILE=data/I_80_test.json -TIM_DATA_FILE=data/I_80_test_TIMS.json PPM_CONTAINER_NAME=test_ppm_instance PPM_IMAGE_TAG=do-kafka-test-ppm-image PPM_IMAGE_NAME=jpo-cvdp_ppm @@ -38,7 +37,6 @@ setup() { echo "KAFKA_CONTAINER_NAME is resolved dynamically" echo "MAP_FILE: $MAP_FILE" echo "BSM_DATA_FILE: $BSM_DATA_FILE" - echo "TIM_DATA_FILE: $TIM_DATA_FILE" echo "PPM_CONTAINER_NAME: $PPM_CONTAINER_NAME" echo "PPM_IMAGE_TAG: $PPM_IMAGE_TAG" echo "PPM_IMAGE_NAME: $PPM_IMAGE_NAME" @@ -63,12 +61,8 @@ waitForKafkaToCreateTopics() { allTopicsCreated=true if [ $(echo $ltopics | grep "topic.FilteredOdeBsmJson" | wc -l) == "0" ]; then allTopicsCreated=false - elif [ $(echo $ltopics | grep "topic.FilteredOdeTimJson" | wc -l) == "0" ]; then - allTopicsCreated=false elif [ $(echo $ltopics | grep "topic.OdeBsmJson" | wc -l) == "0" ]; then allTopicsCreated=false - elif [ $(echo $ltopics | grep "topic.OdeTimJson" | wc -l) == "0" ]; then - allTopicsCreated=false fi if [ $allTopicsCreated == true ]; then @@ -94,57 +88,38 @@ run_tests() { echo "--- File Being Used ---" echo $MAP_FILE echo $BSM_DATA_FILE - echo $TIM_DATA_FILE echo "-----------------" - numberOfTests=10 + numberOfTests=6 echo -e $YELLOW"Test 1/$numberOfTests"$NC - ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c1.properties $BSM_DATA_FILE BSM 0 + ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c1.properties $BSM_DATA_FILE 0 echo "" echo "" echo -e $YELLOW"Test 2/$numberOfTests"$NC - ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c2.properties $BSM_DATA_FILE BSM 10 + ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c2.properties $BSM_DATA_FILE 10 echo "" echo "" echo -e $YELLOW"Test 3/$numberOfTests"$NC - ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c3.properties $BSM_DATA_FILE BSM 18 + ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c3.properties $BSM_DATA_FILE 18 echo "" echo "" echo -e $YELLOW"Test 4/$numberOfTests"$NC - ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c4.properties $BSM_DATA_FILE BSM 23 + ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c4.properties $BSM_DATA_FILE 23 echo "" echo "" echo -e $YELLOW"Test 5/$numberOfTests"$NC - ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c5.properties $BSM_DATA_FILE BSM 33 + ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c5.properties $BSM_DATA_FILE 33 echo "" echo "" echo -e $YELLOW"Test 6/$numberOfTests"$NC - ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c6.properties $BSM_DATA_FILE BSM 43 - echo "" - echo "" - - echo -e $YELLOW"Test 7/$numberOfTests"$NC - ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c1.properties $TIM_DATA_FILE TIM 0 - echo "" - echo "" - - echo -e $YELLOW"Test 8/$numberOfTests"$NC - ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c2.properties $TIM_DATA_FILE TIM 10 - echo "" - echo "" - - echo -e $YELLOW"Test 9/$numberOfTests"$NC - ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c3.properties $TIM_DATA_FILE TIM 18 + ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c6.properties $BSM_DATA_FILE 43 echo "" echo "" - - echo -e $YELLOW"Test 10/$numberOfTests (2 tests in one)"$NC - ./test-scripts/standalone_multi.sh $MAP_FILE config/bsm-test/c6.properties config/tim-test/c3.properties $BSM_DATA_FILE $TIM_DATA_FILE 48 23 } cleanup() { diff --git a/docker-compose-confluent-cloud.yml b/docker-compose-confluent-cloud.yml index 0a358b7e..3b376183 100644 --- a/docker-compose-confluent-cloud.yml +++ b/docker-compose-confluent-cloud.yml @@ -1,6 +1,5 @@ -version: '2' services: - ppmbsm: + ppm: build: context: . dockerfile: Dockerfile @@ -15,23 +14,5 @@ services: PPM_LOG_TO_CONSOLE: ${PPM_LOG_TO_CONSOLE} RPM_DEBUG: ${RPM_DEBUG} PPM_LOG_LEVEL: ${PPM_LOG_LEVEL} - volumes: - - ${DOCKER_SHARED_VOLUME}:/ppm_data - - ppmtim: - build: - context: . - dockerfile: Dockerfile - environment: - DOCKER_HOST_IP: ${DOCKER_HOST_IP} - KAFKA_TYPE: ${KAFKA_TYPE} - CONFLUENT_KEY: ${CONFLUENT_KEY} - CONFLUENT_SECRET: ${CONFLUENT_SECRET} - PPM_CONFIG_FILE: ppmTim.properties - REDACTION_PROPERTIES_PATH: ${REDACTION_PROPERTIES_PATH} - PPM_LOG_TO_FILE: ${PPM_LOG_TO_FILE} - PPM_LOG_TO_CONSOLE: ${PPM_LOG_TO_CONSOLE} - RPM_DEBUG: ${RPM_DEBUG} - PPM_LOG_LEVEL: ${PPM_LOG_LEVEL} volumes: - ${DOCKER_SHARED_VOLUME}:/ppm_data \ No newline at end of file diff --git a/docker-compose-kafka.yml b/docker-compose-kafka.yml index 7d9a61f0..610badd1 100644 --- a/docker-compose-kafka.yml +++ b/docker-compose-kafka.yml @@ -1,4 +1,3 @@ -version: '2' services: zookeeper: image: wurstmeister/zookeeper @@ -11,6 +10,6 @@ services: environment: KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP} KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_CREATE_TOPICS: "topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1,topic.OdeTimJson:1:1,topic.FilteredOdeTimJson:1:1" + KAFKA_CREATE_TOPICS: "topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1" volumes: - /var/run/docker.sock:/var/run/docker.sock \ No newline at end of file diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml deleted file mode 100644 index 41992658..00000000 --- a/docker-compose-standalone.yml +++ /dev/null @@ -1,54 +0,0 @@ -version: '2' -services: - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181:2181" - - kafka: - image: wurstmeister/kafka - ports: - - "9092:9092" - environment: - KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP} - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_CREATE_TOPICS: "topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1,topic.OdeTimJson:1:1,topic.FilteredOdeTimJson:1:1" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ppm_bsm: - build: - context: . - dockerfile: Dockerfile - environment: - DOCKER_HOST_IP: ${DOCKER_HOST_IP} - KAFKA_TYPE: ${KAFKA_TYPE} - CONFLUENT_KEY: ${CONFLUENT_KEY} - CONFLUENT_SECRET: ${CONFLUENT_SECRET} - PPM_CONFIG_FILE: ppmBsm.properties - REDACTION_PROPERTIES_PATH: ${REDACTION_PROPERTIES_PATH} - PPM_LOG_TO_FILE: ${PPM_LOG_TO_FILE} - PPM_LOG_TO_CONSOLE: ${PPM_LOG_TO_CONSOLE} - RPM_DEBUG: ${RPM_DEBUG} - PPM_LOG_LEVEL: ${PPM_LOG_LEVEL} - depends_on: - - kafka - volumes: - - ${DOCKER_SHARED_VOLUME}:/ppm_data - - ppm_tim: - build: - context: . - dockerfile: Dockerfile - environment: - DOCKER_HOST_IP: ${DOCKER_HOST_IP} - PPM_CONFIG_FILE: ppmTim.properties - REDACTION_PROPERTIES_PATH: ${REDACTION_PROPERTIES_PATH} - PPM_LOG_TO_FILE: ${PPM_LOG_TO_FILE} - PPM_LOG_TO_CONSOLE: ${PPM_LOG_TO_CONSOLE} - RPM_DEBUG: ${RPM_DEBUG} - PPM_LOG_LEVEL: ${PPM_LOG_LEVEL} - depends_on: - - kafka - volumes: - - ${DOCKER_SHARED_VOLUME}:/ppm_data \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index e4d530c1..33d4bfcd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2' services: zookeeper: image: wurstmeister/zookeeper @@ -11,14 +10,14 @@ services: environment: KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP} KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_CREATE_TOPICS: "topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1,topic.OdeTimJson:1:1,topic.FilteredOdeTimJson:1:1" + KAFKA_CREATE_TOPICS: "topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1" volumes: - /var/run/docker.sock:/var/run/docker.sock ppm: build: context: . - dockerfile: Dockerfile.standalone + dockerfile: Dockerfile ports: - "8080:8080" - "9090:9090" diff --git a/docker-test/do_tim_test.sh b/docker-test/do_tim_test.sh deleted file mode 100755 index 6df20b25..00000000 --- a/docker-test/do_tim_test.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# This script produces and consumes messages from Kafka topics. It reads a JSON file containing raw -# TIM data, processes it with a Python script, and then sends the output to a Kafka topic. Then it -# consumes the filtered messages from the Kafka topic, using a specified offset, and checks if -# any messages were received. If no messages were received after a certain number of attempts, the -# script exits with an error message. Otherwise, the script exits with a success message. - -export LD_LIBRARY_PATH=/usr/local/lib - -RED='\033[0;31m' -GREEN='\033[0;32m' -NC='\033[0m' # No Color - -broker=$DOCKER_HOST_IP:9092 - -echo "**************************" -echo "Producing Raw TIMs..." -echo "**************************" -cat /ppm_data/tim_test.json | /cvdi-stream/docker-test/test_in.py | /cvdi-stream-build/kafka-test/kafka_tool -P -b $broker -p 0 -t topic.OdeTimJson 2> priv.err - -# Start the DI consumer. -offset=$1 - -echo "**************************" -echo "Consuming Filtered TIMs at offset "$offset "..." -echo "**************************" - -attempts=0 -max_attempts=5 -while true; do - attempts=$((attempts+1)) - - timeout 5 /cvdi-stream-build/kafka-test/kafka_tool -C -b $broker -p 0 -t topic.FilteredOdeTimJson -e -o $offset 2> con.err | /cvdi-stream/docker-test/test_out.py > tmp.out - if [[ $? != 0 ]]; then - echo "Error: Kafka consumer timed out." - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo -e $RED"TEST FAILED!"$NC - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" - exit 1 - fi - - lines=$(cat tmp.out | wc -l) - if [[ $lines != "0" ]]; then - cat tmp.out - break - else - if [[ $attempts > $max_attempts ]]; then - echo "No data received after $max_attempts attempts. Exiting..." - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo -e $RED"TEST FAILED!"$NC - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" - exit 1 - fi - fi -done -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" -echo -e $GREEN"TEST PASSED!"$NC -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" \ No newline at end of file diff --git a/docker-test/test_in.py b/docker-test/test_in.py index 743713ed..d65fe3d7 100755 --- a/docker-test/test_in.py +++ b/docker-test/test_in.py @@ -6,39 +6,26 @@ import sys def print_bsm_data(d): - if d['metadata']['payloadType'] == 'us.dot.its.jpo.ode.model.OdeTimPayload': - speed = d['metadata']['receivedMessageDetails']['locationData']['speed'] - lat = d['metadata']['receivedMessageDetails']['locationData']['latitude'] - lng = d['metadata']['receivedMessageDetails']['locationData']['longitude'] + id_ = d['payload']['data']['coreData']['id'] + speed = d['payload']['data']['coreData']['speed'] + lat = d['payload']['data']['coreData']['position']['latitude'] + lng = d['payload']['data']['coreData']['position']['longitude'] - print('Producing TIMS with speed={}, position={}, {}'.format(speed, lat, lng), file=sys.stderr) + if not d['payload']['data']['coreData']['size']: + print('Consuming BSM with ID={}, speed={}, position={}, {}'.format(id_, speed, lat, lng)) return - elif d['metadata']['payloadType'] == 'us.dot.its.jpo.ode.model.OdeBsmPayload': - id_ = d['payload']['data']['coreData']['id'] - speed = d['payload']['data']['coreData']['speed'] - lat = d['payload']['data']['coreData']['position']['latitude'] - lng = d['payload']['data']['coreData']['position']['longitude'] - if not d['payload']['data']['coreData']['size']: - print('Consuming BSM with ID={}, speed={}, position={}, {}'.format(id_, speed, lat, lng)) + length = 0 + width = 0 - return + if d['payload']['data']['coreData']['size']['length']: + length = d['payload']['data']['coreData']['size']['length'] - length = 0 - width = 0 + if d['payload']['data']['coreData']['size']['width']: + width = d['payload']['data']['coreData']['size']['width'] - if d['payload']['data']['coreData']['size']['length']: - length = d['payload']['data']['coreData']['size']['length'] - - if d['payload']['data']['coreData']['size']['width']: - width = d['payload']['data']['coreData']['size']['width'] - - print('Producing BSM with ID={}, speed={}, position={}, {}, size=l:{}, w:{}'.format(id_, speed, lat, lng, length, width), file=sys.stderr) - - return - - raise IOError() + print('Producing BSM with ID={}, speed={}, position={}, {}, size=l:{}, w:{}'.format(id_, speed, lat, lng, length, width), file=sys.stderr) for l in sys.stdin: diff --git a/docker-test/test_out.py b/docker-test/test_out.py index 326133bf..3a4d47ce 100755 --- a/docker-test/test_out.py +++ b/docker-test/test_out.py @@ -6,39 +6,26 @@ import sys def print_bsm_data(d): - if d['metadata']['payloadType'] == 'us.dot.its.jpo.ode.model.OdeTimPayload': - speed = d['metadata']['receivedMessageDetails']['locationData']['speed'] - lat = d['metadata']['receivedMessageDetails']['locationData']['latitude'] - lng = d['metadata']['receivedMessageDetails']['locationData']['longitude'] + id_ = d['payload']['data']['coreData']['id'] + speed = d['payload']['data']['coreData']['speed'] + lat = d['payload']['data']['coreData']['position']['latitude'] + lng = d['payload']['data']['coreData']['position']['longitude'] - print('Consuming TIMS with speed={}, position={}, {}'.format(speed, lat, lng)) + if not d['payload']['data']['coreData']['size']: + print('Consuming BSM with ID={}, speed={}, position={}, {}'.format(id_, speed, lat, lng)) return - elif d['metadata']['payloadType'] == 'us.dot.its.jpo.ode.model.OdeBsmPayload': - id_ = d['payload']['data']['coreData']['id'] - speed = d['payload']['data']['coreData']['speed'] - lat = d['payload']['data']['coreData']['position']['latitude'] - lng = d['payload']['data']['coreData']['position']['longitude'] - if not d['payload']['data']['coreData']['size']: - print('Consuming BSM with ID={}, speed={}, position={}, {}'.format(id_, speed, lat, lng)) + length = 0 + width = 0 - return + if d['payload']['data']['coreData']['size']['length']: + length = d['payload']['data']['coreData']['size']['length'] - length = 0 - width = 0 + if d['payload']['data']['coreData']['size']['width']: + width = d['payload']['data']['coreData']['size']['width'] - if d['payload']['data']['coreData']['size']['length']: - length = d['payload']['data']['coreData']['size']['length'] - - if d['payload']['data']['coreData']['size']['width']: - width = d['payload']['data']['coreData']['size']['width'] - - print('Consuming BSM with ID={}, speed={}, position={}, {}, size=l:{}, w:{}'.format(id_, speed, lat, lng, length, width)) - - return - - raise IOError() + print('Consuming BSM with ID={}, speed={}, position={}, {}, size=l:{}, w:{}'.format(id_, speed, lat, lng, length, width)) for l in sys.stdin: try: diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..6092ad9c --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +*/*/*.bkp \ No newline at end of file diff --git a/docs/Release_notes.md b/docs/Release_notes.md index 6dff78b0..824088fa 100644 --- a/docs/Release_notes.md +++ b/docs/Release_notes.md @@ -1,8 +1,19 @@ Jpo-cvdp Release Notes ---------------------------- -Version 1.3.0, released February 2024 +Version 1.4.0, released September 2024 +---------------------------------------- +### **Summary** +The changes for the jpo-cvdp 1.4.0 release involve the addition of a 'Supported Message Types' section to the README, the removal of TIM support since TIMs do not contain personally-identifiable information, and revisions to the documentation for accuracy & clarity. + +Enhancements in this release: +- CDOT PR 44: Added 'Supported Message Types' section to README +- CDOT PR 45: Removed TIM support since TIMs do not contain personally-identifiable information +- CDOT PR 46: Revised documentation for accuracy & clarity + +Version 1.3.0, released February 2024 +---------------------------------------- ### **Summary** The changes for the jpo-cvdp 1.3.0 release involve the optimization of dockerfiles, addition of dockerhub image documentation & some QoL changes to the `do_kafka_test.sh` script. diff --git a/docs/configuration.md b/docs/configuration.md index 586cac4f..4b709be5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,15 +1,27 @@ -# PPM Operation +# Configuration +## Table of Contents +1. [PPM Operation](#ppm-operation) +2. [PPM Command Line Options](#ppm-command-line-options) +3. [PPM Deployment](#ppm-deployment) +4. [PPM Kafka Limitations](#ppm-kafka-limitations) +5. [Multiple PPM Instances with Different Configurations](#multiple-ppm-instances-with-different-configurations) +6. [PPM Logging](#ppm-logging) +7. [PPM Configuration](#ppm-configuration) +8. [Map Files](#map-files) +9. [Environment Variables](#environment-variables) -The messages suppressed and sanitized by the PPM are documented [here](https://github.com/usdot-jpo-ode/jpo-ode/blob/develop/docs/metadata_standards.md). +## PPM Operation -The PPM suppresses BSMs and TIMs message and redacts BSM ID fields based on several conditions. These conditions are determined by a set of configuration parameters. The following conditions will result in a message being suppressed, or deleted, from the stream. +The messages suppressed and sanitized by the PPM are documented [here](../README.md#supported-message-types). + +The PPM suppresses BSMs and redacts BSM ID fields based on several conditions. These conditions are determined by a set of configuration parameters. The following conditions will result in a message being suppressed, or deleted, from the stream. 1. Message JSON record cannot be parsed. 2. Message speed is outside of prescribed limits. 3. Message location is outside of a prescribed geofence. -4. BSM TemporaryID can be redacted (rendered indistinct). +4. BSM TemporaryID cannot be redacted (rendered indistinct). -## PPM Command Line Options +### PPM Command Line Options The PPM can be started by specifying only the configuration file. Command line options are also available. **Command line options override parameters specified in the configuration file.** The following command line options are available: @@ -31,7 +43,7 @@ line options override parameters specified in the configuration file.** The foll -m | --mapfile : The path to the map file to use to build the geofence. ``` -# PPM Deployment +## PPM Deployment Once the PPM is [installed and configured](installation.md) it operates as a background service. The PPM can be started before or after other services. If started before the other services, it may produce some error messages while it waits @@ -43,7 +55,7 @@ $ ./ppm -c We recommend reviewing the [testing documentation](testing.md) for more details on running the PPM. -# PPM Kafka Limitations +## PPM Kafka Limitations With regard to the Apache Kafka architecture, each PPM process does **not** provide a way to take advantage of Kafka's scalable architecture. In other words, each PPM process will consume data from a single Kafka topic and a single partition within @@ -51,13 +63,13 @@ that topic. One way to consume topics with multiple partitions is to launch one configuration file will allow you to designate the partition. In the future, the PPM may be updated to automatically handle multiple partitions within a single topic. -# Multiple PPM Instances with Different Configurations +## Multiple PPM Instances with Different Configurations Nothing prevents a users from launching multiple PPM instances where each uses a different configuration file. This strategy would allow various degrees of privacy protection. It would also allow a user to publish various versions of the data to different "filtered" topics. -# PPM Logging +## PPM Logging PPM operations are optionally logged to the console or a file. The file is a rotating log file, i.e., a set number of log files will be used to record the PPM's information. By default, the file is in a `logs` directory from where the ACM is launched and the file is @@ -84,7 +96,7 @@ with a date and time stamp and the level of the log message. [170613 12:25:47.443150] [info] BSM [SUPPRESSED-speed]: (ON-VBL--,36712,41.116496,-104.888494,1.000000) ``` -# PPM Configuration +## PPM Configuration The PPM configuration file is a text file with a specific format. It can be used to configure Kafka as well as the PPM. Comments can be added to the configuration file by starting a line with the '#' character. Configuration lines consist @@ -103,7 +115,7 @@ Example configuration files can be found in the [jpo-cvdp/config](../config) dir The details of the settings and how they affect the function of the PPM follow: -## Sanitization Flag +### Sanitization Flag The current JSON data object sent to the PPM and published by the PPM contains the following three named components: parts: @@ -117,25 +129,24 @@ if the message is published. The `metadata:sanitized` element is changed to `tru all messages published by the PPM should have a `metadata:sanitized` value of `true`. The `payload` component of the BSM has a `data` object containing the `coreData` object that is analyzed by the PPM -for features that may cause it to be suppressed. The same analysis is done on the `receivedDetails:loction` object in the TIMS -`metadata` component. Note that the `payload` for TIMS is not inspected. +for features that may cause it to be suppressed. The JSON format published by the PPM follows the format received. It may be completely suppressed or certain fields may -be modifed as described in this second and the sections that follow. +be modified as described in this second and the sections that follow. -## Velocity Filtering +### Velocity Filtering - `privacy.filter.velocity` : enables or disables message filtering based on the speed within the message. - `ON` : enables message filtering. - Any other value : disables message filtering. -- `privacy.filter.velocity.min` : *When velocity fitering is enabled*, messages having velocities below this value will be +- `privacy.filter.velocity.min` : *When velocity filtering is enabled*, messages having velocities below this value will be suppressed. The units are in meters per second. -- `privacy.filter.velocity.max` : *When velocity fitering is enabled*, messages having velocities above this value will be +- `privacy.filter.velocity.max` : *When velocity filtering is enabled*, messages having velocities above this value will be suppressed. The units are in meters per second. -## BSM Identifier Redaction +### BSM Identifier Redaction If required, the `TemporaryID` field in the BSM can be redacted and replaced with a randomly chosen identifier. The following configuration parameters control identifier redaction. @@ -156,7 +167,7 @@ control identifier redaction. - Similar to the `privacy.redaction.id.value`, these are 4 hexadecimal-encoded bytes. - More than one id can be specified by separating them by commas. -## BSM Vehicle Size Redaction +### BSM Vehicle Size Redaction If required, the `VehicleLength` and `VehicleWidth` fields in the BSM can be redacted and replaced with a **0** value. The following configuration parameters control vehicle size redaction. @@ -165,14 +176,14 @@ control vehicle size redaction. - `ON` : enables redaction - Any other value : disables redaction. -## Geofencing +### Geofencing Messages can be suppressed based on latitude and longitude attributes. If this -capability is turned one through the configuration file, each edge defined in the +capability is turned on through the configuration file, each edge defined in the map file is used to infer a *component* geofence that surrounds that segment of the -road. The image below illustrates how a *rectange* is drawn to form the segment's +road. The image below illustrates how a *rectangle* is drawn to form the segment's geofence. The aforementioned edge attributes and PPM configuration parameters -determine the size of the rectange. +determine the size of the rectangle. ![Road Segment Geofence Dimensions](graphics/geofence-dimensions.png) @@ -187,7 +198,7 @@ determine the size of the rectange. of the controls that determines the size of the component geofences that surround road segments. See the [Map Files](#geofencing) section. -### Geofence Region Boundaries +#### Geofence Region Boundaries Geofence Boundary Configuration Parameters: The geofence is stored in a geographically-defined data structured called a quadtree. The following bounding box coordinates define the quadtree's region. The data that is stored in this data @@ -202,7 +213,7 @@ instead of having to modify the mapfile. - `privacy.filter.geofence.ne.lat` : The latitude of the upper-right corner of the quadtree region. - `privacy.filter.geofence.ne.lon` : The longitude of the upper-right corner of the quadtree region. -## ODE Kafka Interface +### ODE Kafka Interface - `privacy.topic.producer` : The Kafka topic name where the PPM will write the filtered messages. **The name is case sensitive.** @@ -217,7 +228,7 @@ instead of having to modify the mapfile. themselves with a consumer group name, and each record published to a topic is delivered to one consumer instance within each subscribing consumer group. Consumer instances can be in separate processes or on separate machines. **Due to the way the kafka library - internally updates its topic offsets, the group ID must be unique for each the topic.** + internally updates its topic offsets, the group ID must be unique for each topic.** - `privacy.kafka.partition` : The partition(s) that this PPM will consume records from. A Kafka topic can be divided, or partitioned, into several "parallel" streams. A topic may have many partitions so it can handle an arbitrary @@ -227,7 +238,7 @@ instead of having to modify the mapfile. - `compression.type` : The type of compression to use for writing to Kafka topics. Currently, this should be set to none. -# Map Files +## Map Files The map file is used to define the geofence. It defines a set of shapes, one per line. For road geofence use, the edge shape is used. The map file for the @@ -254,5 +265,21 @@ This file has four comma-separated elements: For the WYDOT use case, WYDOT provided a set of edge definitions for I-80 that were converted into the above format. -## See Also: Data & Config Files -More information on config files can be found in the [Data & Config Files](../README.md#data--config-files) section of the README. \ No newline at end of file +### See Also: Data & Config Files +More information on config files can be found in the [Data & Config Files](../README.md#data--config-files) section of the README. + +## Environment Variables +The following table lists the environment variables that can be used to configure the PPM: +| Variable | Description | +|----------|-------------| +| `DOCKER_HOST_IP` | The IP address of the Docker host. | +| `DOCKER_SHARED_VOLUME` | The path to the shared volume where the map file and configuration file are located. | +| `PPM_CONFIG_FILE` | The path to the PPM configuration file. | +| `REDACTION_PROPERTIES_PATH` | The path to the redaction properties file. | +| `PPM_LOG_TO_FILE` | The path to the log file. | +| `PPM_LOG_TO_CONSOLE` | The path to the console log file. | +| `PPM_LOG_LEVEL` | The log level. | +| `RPM_DEBUG` | When set to true, the Redaction Properties Manager will print debug messages to a file. | +| `KAFKA_TYPE` | The type of Kafka broker. If not set, a local Kafka broker will be used. | +| `CONFLUENT_KEY` | The Confluent key. Only used if `KAFKA_TYPE` is set to `CONFLUENT`. | +| `CONFLUENT_SECRET` | The Confluent secret. Only used if `KAFKA_TYPE` is set to `CONFLUENT`. | diff --git a/docs/diagrams/class-usage/PPM Class Usage.drawio b/docs/diagrams/class-usage/PPM Class Usage.drawio index 64f6f9fa..48d70956 100644 --- a/docs/diagrams/class-usage/PPM Class Usage.drawio +++ b/docs/diagrams/class-usage/PPM Class Usage.drawio @@ -1 +1,173 @@ -7VxRd6o4EP41nrP74B5JQPSxWm27t73rvd617VNPKqnSAqEQq95fv0FAIKHU2kJk7VPJkCD5vsnMZDK0Afv26sxD7vyKGNhqgJaxasDTBgBAaWnsTyBZh5K22g0FM880QpGSCMbmbxwJW5F0YRrYz3SkhFjUdLPCKXEcPKUZGfI8ssx2eyBW9lddNMOCYDxFlii9Ng06D6UdoCfyc2zO5vEvK+1ofjaKO0cz8efIIMuUCA4asO8RQsMre9XHVgBejMv1xfraunxqn/39w39G//a+/fo+aYYPG75nyHYKHnbo3o+mmj+50YbG5Z07GoG76/vhzVn86BdkLSK8Bs7CPkf+PJozXcdAemThGDh4WKsBe8u5SfHYRdPg7pKpDpPNqW2xlsIuH0zL6hOLeJuxsN8faEP2Yr0dZxK/FvYoXqV4jGZ2homNqbdmXaK7sKWGQyIthe2ItGXCuRITOU/xHfdDkZrNto9OoGQXEZrvQBYKyF4YP7GBppSBUi9sY9xiC9CSja0qYDvBFpmadD00LYprhq/S1g4MX03Atze+qheoTQCyqMK4XQWqVn/xAGeg0713bXPyT/vMhaQJBASxwVxV1CQenZMZcZA1SKS9LMZJn0tC3AjZR0zpOvK7aEFJFne8MulNMPwvNv+weZu6dbqKHr1prOOGw+YbjtLi5m36XjJs04rHvY9Mnyy8KS5Qw8gRU+TNMC0ANnpegGahanjYQtR8yQYGeTRvhp54HlqnOrjEdKifevIoEKSWsZI1kxrg/DHXH2pF3dlF+AKJvm1nsr8KQqkqmNK/lDrmq2D5qlRk+SSrUpPzuFDXClVJ6K9p5euSLtec7W7NJKmS+tmq9CHn05HLlv5/cD5Fcb5s56NnLYDaecP5dAr7Zy2GMFrlRmsaFzOFqEWjOM39BNOjKDK0WZKCfXp0s6sd+T2+O9Hh7Tlmi+zGVskLfdQPKYIoDiC2ZkTJGJHEpkgzI7m4goMwI02VCyS6xWaE719oRvZY/UWQZnen58gxrLrt/DkzWmnSKn9/IMWwpjy+/i6XfwDRoKLuaMa7B7HAARcnQKV4gfNxBde/nJ2FmJT+iVzTePSJU88MalPbIYOqV7rQ5fjxZKGDvRa60tg9ANh/oS/BuKsub4E21CYX183Tl7U5jg5XZK1zMQTPZo8UPgIP51NaBA6PYPf/ITUoJ5EkBmB81hGouynCuyPDNvc7+huRIX+qwQ0oyXMAwXXgFcWO4Qvqykw3zWqXTz3yhGO/4BAHc64iEiHLnDmsOWUaxSJO2AscgTlF1kl0wzYNY6PweW4ouwjK8ji8n9+GkSmP085xOLAsh6OI56ELHx8dLa+t1gpoyd12tgRWZNj1/W10vq5psoz0x5aIeOx6hEukCbkdR4VLJJ+W9hctCeRxogfKZkX/YoXd7WaDLFWTTUvnixbRzcOOWHtSLS1izvQIaVE417I9xpJFCxAL2QYrZLsWA9zxFzbu3wsUHXaii09pS89oA9FPXK2D+tYRYvpJTaa9Xu1R3uYOq4C5edLXhqNT7blFJ9bUfT5f3Tzm7COO0L5wVThV7rlzSRHz6EdISpPLhGx9szRf3BIYqGCL7c+RG9x5sPDqJPisooSjsCj2e7uiQVpplDHWv3de1McrY2kPyGR1sQST+Og9fdy0OWVivmHkETdQaexfIQfN6na+3AS7lJaXdfCUC7YY8RyjTeID0VZ1+4NcVg6oqkeXXRdcZCNkl+eAfL3ZtS44rtN7/dDmlcK+zzq0KSp9SpmEkWtfklkNzS14Ox1TWlyei+2XuQ146XLLJudToLLMbWEAlFb5Uc0+sOJzXJXuQQvjzxSqvwixagprFTDmVjjIKQHcJxb4WIkv58vLrREpyE1VsO3JfUtwODy/8S1Y7XlWZfIspua+oYcnFGWYSw2xBLO3K/ivu3KFT7IpFX7Wm4tvDQtbyuCFK3Gt8MQrl5UaBr4VsKLokmkRS1z+sJHpMNHUQr7/55ERBLnvZ5ScZZOXntuDH9ZM/lVNuHVP/uEPHPwH \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/diagrams/class-usage/PPM Class Usage.drawio.png b/docs/diagrams/class-usage/PPM Class Usage.drawio.png index 9741bdcd..d86749bd 100644 Binary files a/docs/diagrams/class-usage/PPM Class Usage.drawio.png and b/docs/diagrams/class-usage/PPM Class Usage.drawio.png differ diff --git a/docs/dockerhub.md b/docs/dockerhub.md index 931d8a3d..d474afde 100644 --- a/docs/dockerhub.md +++ b/docs/dockerhub.md @@ -46,7 +46,7 @@ services: environment: KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP} KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_CREATE_TOPICS: "topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1,topic.OdeTimJson:1:1,topic.FilteredOdeTimJson:1:1" + KAFKA_CREATE_TOPICS: "topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1" volumes: - /var/run/docker.sock:/var/run/docker.sock diff --git a/docs/installation.md b/docs/installation.md index b5d28062..052bbf7d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,34 +1,17 @@ # Installation and Setup +## Table of Contents +1. [Docker Installation](#docker-installation) +1. [Manual Installation](#manual-installation) +1. [CDOT Integration with K8s](#cdot-integration-with-k8s) -The following instructions represent the "hard" way to install and test the PPM. A docker image can be built to make -this easier: [Using the Docker Container](#using-the-docker-container). *The directions that follow were developed for a clean installation of Ubuntu.* - -## 1. Install [Git](https://git-scm.com/) - -```bash -$ sudo apt install git -``` - -## 2. Install Oracle’s Java - -```bash -$ sudo add-apt-repository -y ppa:webupd8team/java -$ sudo apt update -$ sudo apt install oracle-java8-installer -y -$ sudo java -version -``` - -## 3. Install [CMake](https://cmake.org) to build the PPM - -```bash -$ sudo apt install cmake -``` - -## 4. Install [Docker](https://www.docker.com) +## Docker Installation +### 1. Install [Docker](https://www.docker.com) - When following the website instructions, setup the Docker repos and follow the Linux post-install instructions. - The CE version seems to work fine. - [Docker installation instructions](https://docs.docker.com/engine/installation/linux/ubuntu/#install-using-the-repository) + +#### ORNL Specific Docker Configuration - *ORNL specific, but may apply to others with organizational security* - Correct for internal Google DNS blocking - As root (`$ sudo su`), create a `daemon.json` file in the `/etc/docker` directory that contains the following information: @@ -36,7 +19,7 @@ $ sudo apt install cmake { "debug": true, "default-runtime": "runc", - "dns": ["160.91.126.23","160.91.126.28”], + "dns": ["160.91.126.23","160.91.126.28"], "icc": true, "insecure-registries": [], "ip": "0.0.0.0", @@ -49,88 +32,71 @@ $ sudo apt install cmake ``` - NOTE: The DNS IP addresses are ORNL specific. -## 5. Restart the docker daemon to consume the new configuration file. +Be sure to restart the docker daemon to consume the new configuration file. ```bash $ service docker stop $ service docker start ``` -## 6. Check the configuration using the command below to confirm the updates above are taken if needed: +Check the configuration using the command below to confirm the updates above are taken if needed: ```bash $ docker info ``` -## 7. Install Docker Compose -- Comprehensive instructions can be found on this [website](https://www.digitalocean.com/community/tutorials/how-to-install-docker-compose-on-ubuntu-16-04) -- Follow steps 1 and 2. - -## 8. Create a base directory from which to install all the necessary components to test the PPM. +### 2. Configure environment variables +Configure the environment variables for the PPM to communicate with the Kafka instance. Copy or rename the `sample.env` file to `.env`. ```bash -$ export BASE_PPM_DIR=~/some/dir/you/want/to/put/this/stuff +$ cp sample.env .env ``` -## 9. Install [`kafka-docker`](https://github.com/wurstmeister/kafka-docker) so kafka and zookeeper can run in a separate container. - -- Get your host IP address. The address is usually listed under an ethernet adapter, e.g., `en`. +Edit the `.env` file to include the necessary information. ```bash -$ ifconfig -$ export DOCKER_HOST_IP= +$ vi .env ``` -- Get the kafka and zookeeper images. -```bash -$ cd $BASE_PPM_DIR -$ git clone https://github.com/wurstmeister/kafka-docker.git -$ cd kafka-docker -$ vim docker-compose.yml // Set karka: ports: to 9092:9092 -``` -- The `docker-compose.yml` file may need to be changed; the ports for kafka should be 9092:9092. -- Startup the kafka and zookeeper containers and make sure they are running. +For more information on the environment variables, see the 'Environment Variables' section in the [configuration.md](configuration.md) file. + +### 3. Spin up Kafka & the PPM in Docker +To spin up the PPM and Kafka in Docker, use the following commands: ```bash -$ docker-compose up --no-recreate -d -$ docker-compose ps +docker compose up --build ``` -- **When you want to stop kafka and zookeeper, execute the following commands.** + +## Manual Installation +The following instructions represent the "hard" way to install and test the PPM. A docker image can be built to make +this easier: [Using the Docker Container](#using-the-docker-container). *The directions that follow were developed for a clean installation of Ubuntu.* + +### 1. Install [Git](https://git-scm.com/) ```bash -$ cd $BASE_PPM_DIR/kafka-docker -$ docker-compose down +$ sudo apt install git ``` -## 10. Download and install the Kafka **binary**. +### 2. Install librdkafka -- The Kafka binary provides a producer and consumer tool that can act as surrogates for the ODE (among other items). -- [Kafka Binary](https://kafka.apache.org/downloads) -- [Kafka Quickstart](https://kafka.apache.org/quickstart) is a very useful reference. -- Move and unpack the Kafka code as follows: +Talking to a Kafka instance, subscribing and producting to topics requires the use of a third party library. We use the librdkafka library as a c/c++ implementation. This can be installed as a package. ```bash -$ cd $BASE_PPM_DIR -$ wget http://apache.claz.org/kafka/0.10.2.1/kafka_2.12-0.10.2.1.tgz // mirror and kafka version may change; check website. -$ tar -xzf kafka_2.12-0.10.2.1.tgz // the kafka version may be different. -$ mv kafka_2.12-0.10.2.1 kafka +$ sudo apt install -y libsasl2-dev +$ sudo apt install -y libsasl2-modules +$ sudo apt install -y libssl-dev +$ sudo apt install -y librdkafka-dev ``` -## 11. Download and install [`librdkafka`](https://github.com/edenhill/librdkafka), the C++ Kafka library we use to build the PPM. +### 3. Install [CMake](https://cmake.org), g++ & make to build the PPM ```bash -$ cd $BASE_PPM_DIR -$ git clone https://github.com/edenhill/librdkafka.git -$ cd librdkafka -$ ./configure -$ make -$ sudo make install +$ sudo apt install cmake +$ sudo apt install g++ +$ sudo apt install make ``` -- **NOTE**: The header files for `librdkafka` should be located in `/usr/local/include/librdkafka` and the libraries - (static and dynamic) should be located in `/usr/local/lib`. If you put them in another location the PPM may not build. - -## 12. Download, Build, and Install the Privacy Protection Module (PPM) +### 4. Download, Build, and Install the Privacy Protection Module (PPM) ```bash $ cd $BASE_PPM_DIR @@ -141,49 +107,21 @@ $ cmake .. $ make ``` -## Additional information +### Additional information - The PPM uses [RapidJSON](https://github.com/miloyip/rapidjson), but it is a header-only library included in the repository. - The PPM uses [spdlog](https://github.com/gabime/spdlog) for logging; it is a header-only library and the headers are included in the repository. - The PPM uses [Catch](https://github.com/philsquared/Catch) for unit testing, but it is a header-only library included in the repository. -# Integrating with the ODE - -## Using the Docker Container - -This will run the PPM module in separate container. First set the required environmental variables. You need to tell the PPM container where the Kafka Docker container is running with the `DOCKER_HOST_IP` variable. Also tell the PPM container where to find the [map file](configuration.md#map-file) and [PPM Configuration file](configuration.md) by setting the `DOCKER_SHARED_VOLUME`: - -```bash -$ export DOCKER_HOST_IP=your.docker.host.ip -$ export DOCKER_SHARED_VOLUME=/your/shared/directory -``` - -Note that the map file and configuration file must be located in the `DOCKER_SHARED_VOLUME` root directory and named -`config.properties` and `road_file.csv` respectively. - -Add the following service to the end of the `docker-compose.yml` file in the `jpo-ode` installation directory. - -```bash - ppm: - build: /path/to/jpo-cvdp/repo - environment: - DOCKER_HOST_IP: ${DOCKER_HOST_IP} - volumes: - - ${DOCKER_SHARED_VOLUME}:/ppm_data -``` - -Start the ODE containers as normal. Note that the topics for raw BSMs must be created ahead of time. - - -# CDOT Integration with K8s +## CDOT Integration with K8s -## Overview +### Overview The Colorado Department of Transportation (CDOT) is deploying the various ODE services within a Kubernetes (K8s) environment. Details of this deployment can be found in the main ODE repository [documentation pages](https://github.com/usdot-jpo-ode/jpo-ode/docs). In general, each submodule image is built as a Docker image and then pushed to the CDOT registry. The images are pulled into containers running within the K8s environment, and additional containers are spun up as load requires. -## CDOT PPM Module Build +### CDOT PPM Module Build Several additional files have been added to this project to facilitate the CDOT integration. These files are: - cdot-scripts/build_cdot.sh - docker-test/ppm_no_map.sh -### Shell Scripts +#### Shell Scripts Two additional scripts have been added to facilitate the CDOT integration. The first, [`ppm_no_map.sh`](../docker-test/ppm_no_map.sh), is modeled after the existing [`ppm.sh`](../docker-test/ppm.sh) script and performs a similar function. This script is used to start the PPM module, but leaves out the hard-coded mapfile name in favor of the properties file configuration. The second script, [`build_cdot.sh`](../cdot-scripts/build_cdot.sh), is used to build the CDOT PPM Docker image, tag the image with a user provided tag, and push that image to a remote repository. This is a simple automation script used to help reduce complexity in the CDOT pipeline. \ No newline at end of file diff --git a/docs/testing.md b/docs/testing.md index fb62c8c3..7be1c472 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,221 +1,120 @@ # Testing the PPM +This document describes how to test the PPM module. The PPM can be tested using unit tests, standalone tests, and Kafka integration tests. -There are several ways to test the capabilities of the PPM. - -- Testing as an individual application within [Docker](#docker-testing) -- Testing as an individual application on the [native client](#testing-on-the-native-client) +## Table of Contents - [Unit Testing](#unit-testing) +- [Standalone Testing](#standalone-testing) +- [Kafka Integration Testing](#kafka-integration-testing) +- [Test Files](#test-files) +- [See Also: Testing/Troubleshooting](#see-also-testingtroubleshooting) -## Test Files - -Several example JSON message test files are in the [jpo-cvdp/data](../data) directory. These files can be edited to generate -your own test cases. Each line in the file should be a well-formed BSM or TIM JSON -object. **Each message should be on a separate line in the file.** **If a JSON object cannot be parsed it is suppressed.** - -## Docker Testing - -To run a series of configuration tests: - - $ ./do_test.sh - -**Running this command for the first time can take awhile, as the dependencies need to be built for the PPM image.** -To run the standalone PPM test using Docker containers, from the jpo-cvdp root directory: - - $ ./start_kafka.sh - -This will build and start the required kafka containers, including the PPM image. -Next run: - - $ ./test-scripts/standalone.sh [MAP_FILE] [CONFIG] [TEST_FILE] [OFFSET] - -Where MAP_FILE is a [map file](configuration.md#map-file), CONFIG is [PPM Configuration file](configuration.md) and TEST_FILE is a [JSON message test file](#test-files). Offset refers to the offset in the filtered message topic; this is where the consumer will look for new output. The default offset is zero (the beginning of the topic), which should for the first time running the test. This will start the PPM Kafka container and use the supplied files to test the BSM filtering. For example, running: - - $ ./test-scripts/standalone.sh data/I_80.edges config/test/I_80_vel_filter.properties data/I_80_test.json - -yields: - -``` -************************** -Running standalone test with data/I_80.edges config/test/I_80_vel_filter.properties data/I_80_test.json -************************** -************************** -Producing Raw BSMs... -************************** -Producing BSM with ID=BEA10000, speed=7.02, position=41.738136, -106.587029 -Producing BSM with ID=BEA10000, speed=7.12, position=41.608656, -109.226824 -Producing BSM with ID=BEA10000, speed=7.16, position=41.311097, -110.512927 -Producing BSM with ID=BEA10000, speed=7.44, position=41.246647, -111.027436 -Producing BSM with ID=BEA10000, speed=7.44, position=41.600371, -106.22341 -Producing BSM with ID=BEA10000, speed=1.78, position=42.29789, -83.72035 -Producing BSM with ID=BEA10000, speed=0.7, position=42.29789, -83.72034 -Producing BSM with ID=BEA10000, speed=6.86, position=42.24576, -83.62337 -Producing BSM with ID=BEA10000, speed=6.84, position=42.24576, -83.62337 -Producing BSM with ID=BEA10000, speed=6.74, position=42.24576, -83.62338 -************************** -Consuming Filtered BSMs at offset 0 ... -************************** -Consuming BSM with ID=BEA10000, speed=7.02, position=41.738136, -106.587029 -Consuming BSM with ID=BEA10000, speed=7.12, position=41.608656, -109.226824 -Consuming BSM with ID=BEA10000, speed=7.16, position=41.311097, -110.512927 -Consuming BSM with ID=BEA10000, speed=7.44, position=41.246647, -111.027436 -Consuming BSM with ID=BEA10000, speed=7.44, position=41.600371, -106.22341 -Consuming BSM with ID=BEA10000, speed=6.86, position=42.24576, -83.62337 -Consuming BSM with ID=BEA10000, speed=6.84, position=42.24576, -83.62337 -Consuming BSM with ID=BEA10000, speed=6.74, position=42.24576, -83.62338 -``` - -## Testing on the Native Client - -The PPM can be tested as a component of the ODE, and it can be tested using a basic Kafka installation on the native -client. - -### ODE Integration Testing - -The PPM is meant to be a module that supports the ODE. The following instructions outline how to perform integration -testing on a single linux installation: - -1. Follow the [ODE installation instructions](https://github.com/usdot-jpo-ode/jpo-ode#documentation) -1. Start a terminal for launching the ODE containers. -1. Set the following environment variables: - -```bash -$ export DOCKER_HOST_IP= -$ export DOCKER_SHARED_VOLUME= -``` - -1. Follow the Deploying ODE Application on a Docker Host directions - -```bash -$ docker-compose up --no-recreate -d -``` - -1. Start a terminal for launching the PPM. - -```bash -$ cd $BASE_PPM_DIR/jpo-cvdp/build -$ ./ppm -c ../config/.properties -``` -1. Open a web browser, and enter the url: `localhost:8080` -1. Click on the **Connect** button. -1. Click on the **Browse** button, find a JSON test file with BSMs (one per line). -1. Click the **Upload** button. - -The BSMs from the file should be listed in the web browser: the BSMs section -lists all the BSMs; the Filtered BSMs section contains the BSMs that were -processed by the PPM and returned back to the ODE. - -### Testing without the ODE - -These instructions describe how to run a collection of BSM test JSON objects through the PPM and examine its operation. -Using *GNU screen* for this work is really handy; you will need several shells. - -Startup `kafka-docker` in its own shell: - -```bash -$ cd $BASE_PPM_DIR/kafka-docker -$ docker-compose up --no-recreate -d // to startup kafka and zookeeper containers -$ docker-compose ps // to check that they are running. -``` - -In another shell, create the simulated ODE produced topic (`j2735BsmRawJson`) and PPM produced topic (`j2735BsmFilteredJson`) - -```bash -$ cd $BASE_PPM_DIR/kafka -$ bin/kafka-topic.sh --create --zookeeper :2181 --replication-factor 1 --partitions 1 --topic j2735BsmRawJson -$ bin/kafka-topic.sh --create --zookeeper :2181 --replication-factor 1 --partitions 1 --topic j2735BsmFilteredJson -``` - -Startup the simulated ODE consumer in the same shell you used to create the topics. - -```bash -$ cd $BASE_PPM_DIR/kafka -$ bin/kafka-console-consumer.sh --bootstrap-server :9092 --topic j2735BsmFilteredJson -``` - -- This process should just wait for input from the PPM module. - -In another shell, startup the PPM - -```bash -$ cd $BASE_PPM_DIR/jpo-cvdp/build -$ ./ppm -c ../config/.properties -``` +## Unit Testing +### Testing On Local Machine +The build_and_run_unit_test.sh script provides an easy method to build and run the PPM's unit tests. It should be noted that this script needs to have the LF end-of-line sequence for it to work. -- At this point the PPM will wait for streaming messages from the simulated ODE - producer. When a message is received output will be generated describing how - the PPM handled the BSM. +#### Steps +1. Pull the project into VSCode +1. Reopen the project in a dev container +1. Open the terminal. +1. Type "sudo su" to run commands as root +1. Type ./build_and_run_unit_tests.sh to run the script -In another shell, send test JSON-encoded BSMs to the PPM. +### Testing Using Docker +1. Start by building the Docker image: ```bash -$ cd $BASE_PPM_DIR/kafka -$ cat $BASE_PPM_DIR/jpo-cvdp/data/ | bin/kafka-console-producer.sh --broker-list :9092 --topic j2735BsmRawJson +$ docker build -t ppm . ``` -- After the messages are written to the `j2735BsmRawJson` topic the shell process should return. - -You can confirm PPM operations in two ways: - -- Open the information log file in an editor and inspect the output. The `[datetimestamp] [info] BSM` messages should describe the actions the PPM is taking based on the input. -- Return to the simulated ODE consumer shell and examine the output JSON; this is more difficult because the JSON does not render well on the screen. - -## Testing All Capabilities - -- To execute the following tests, you will stop the PPM module, if running, with `-C` and then start it up with one of the following `.properties` files. - - `test.allon.properties` - - `test.geofenceonly.properties` - - `test.idredactonly.properties` - - `test.spdonly.properties` -- After starting the PPM module, you will use the shell you created above to send the [testfile](../data/bsm.wy.test.json) to the Kafka producer: +2. Then run unit tests inside the container with the following command: ```bash -$ cat $BASE_PPM_DIR/jpo-cvdp/data/bsm.wy.test.json | bin/kafka-console-producer.sh --broker-list :9092 --topic j2735BsmRawJson +$ docker run -it -e PPM_LOG_TO_CONSOLE=true --name ppm ppm /cvdi-stream-build/ppm_tests ``` -- Open the information log file in an editor and inspect the `info BSM` message (an example of these messages is shown below). The message immediately -to the right of `BSM` indicates whether the message was RETAINED, or passed on to a filtered stream, or SUPPRESSED with the cause. The information in -parenthesis is the TemporaryID, secMark, lat, lon, and speed information in the message; this can be used to test and troubleshoot your configuration. +3. Remove the container: ```bash -[170613 12:30:47.057503] [info] BSM [RETAINED]: (ON-VG---,36710,41.116496,-104.888494,5.000000) -[170613 12:30:47.057566] [info] BSM [RETAINED]: (ON-VG-99,36711,41.116496,-104.888494,5.000000) -[170613 12:30:47.057584] [info] BSM [SUPPRESSED-speed]: (ON-VBL--,36712,41.116496,-104.888494,1.000000) -[170613 12:30:47.057593] [info] BSM [SUPPRESSED-speed]: (ON-VBH--,36713,41.116496,-104.888494,100.000000) -[170613 12:30:47.057623] [info] BSM [RETAINED]: (OFFVG---,36714,41.118110,-104.889282,5.000000) -[170613 12:30:47.057639] [info] BSM [SUPPRESSED-speed]: (OFFVBH--,36715,41.118110,-104.889282,99.000000) -[170613 12:30:47.057670] [info] BSM [RETAINED]: (OFFVGMID,36716,41.141742,-105.361760,9.000000) -[170613 12:30:47.057705] [info] BSM [RETAINED]: (ON-VGTOP,36717,41.143138,-105.361470,9.000000) -[170613 12:30:47.058086] [info] BSM [SUPPRESSED-speed]: (ON-VBTOP,36718,41.143138,-105.361470,1.000000) -[170613 12:30:47.058126] [info] BSM [RETAINED]: (ON-VGBOT,36719,41.140537,-105.362255,9.000000) -[170613 12:30:47.058147] [info] BSM [SUPPRESSED-speed]: (ON-VBBOT,36720,41.140537,-105.362255,50.000000) -[170613 12:30:47.058178] [info] BSM [RETAINED]: (ON-VG---,36721,41.411728,-110.137350,9.000000) -[170613 12:30:47.058213] [info] BSM [RETAINED]: (ON-VG-99,36722,41.411728,-110.137350,9.000000) -[170613 12:30:47.058293] [info] BSM [RETAINED]: (OFFVG---,36723,41.628687,-109.089771,9.000000) -[170613 12:30:47.058451] [info] BSM [RETAINED]: (OFFVG---,36724,41.627758,-109.091004,9.000000) -[170613 12:30:47.058494] [info] BSM [RETAINED]: (O??VG---,36725,41.627672,-109.089390,9.000000) -[170613 12:30:47.064880] [info] BSM [RETAINED]: (ON-VG---,36726,41.627467,-109.089251,9.000000) -[170613 12:30:47.064940] [info] BSM [RETAINED]: (OFFVG---,36727,43.313653,-111.799675,9.000000) +$ docker rm ppm ``` -- The above output will vary depending on which configuration file you use. - -## Unit Testing - -Unit tests are built when the PPM is compiled during installation. Those tests can be run using the following command: - +## Standalone Testing +1. Rename `sample.env` to `.env` + +1. Set `DOCKER_HOST_IP` in the `.env` file to the IP address of your Docker host. This is the IP address of the machine running Docker. + +1. Create a temporary directory called 'ppm_data' in the root of the project directory. + +1. Set `DOCKER_SHARED_VOLUME` in the `.env` file to the full path of the 'ppm_data' directory. + +1. Ensure that the following files are present in the 'ppm_data' directory: + - [fieldsToRedact.txt](../config/fieldsToRedact.txt) + - [ppmBsm.properties](../config/fieldsToSuppress.txt) + - [I_80.edges](../data/I_80.edges) + +1. Spin up Kafka & the PPM + ``` + $ docker compose up -d --build + ``` + +1. View logs of PPM + ``` + $ docker compose logs -f ppm + ``` + +1. Listen to the output topic + ``` + $ kafkacat -b localhost:9092 -t topic.OdeBsmJson -C + ``` + +1. Send a message to the PPM using kafkacat + ``` + $ kafkacat -b localhost:9092 -t topic.OdeBsmJson -P + ``` + + You can now paste a JSON message into the terminal and hit enter. The PPM should log the message and send it to the output topic if it is not suppressed. + + The message immediately to the right of `BSM` indicates whether the message was RETAINED, or passed on to a filtered stream, or SUPPRESSED with the cause. The information in parenthesis is the TemporaryID, secMark, lat, lon, and speed information in the message; this can be used to test and troubleshoot your configuration. + + ```bash + [170613 12:30:47.057503] [info] BSM [RETAINED]: (ON-VG---,36710,41.116496,-104.888494,5.000000) + [170613 12:30:47.057566] [info] BSM [RETAINED]: (ON-VG-99,36711,41.116496,-104.888494,5.000000) + [170613 12:30:47.057584] [info] BSM [SUPPRESSED-speed]: (ON-VBL--,36712,41.116496,-104.888494,1.000000) + [170613 12:30:47.057593] [info] BSM [SUPPRESSED-speed]: (ON-VBH--,36713,41.116496,-104.888494,100.000000) + [170613 12:30:47.057623] [info] BSM [RETAINED]: (OFFVG---,36714,41.118110,-104.889282,5.000000) + [170613 12:30:47.057639] [info] BSM [SUPPRESSED-speed]: (OFFVBH--,36715,41.118110,-104.889282,99.000000) + [170613 12:30:47.057670] [info] BSM [RETAINED]: (OFFVGMID,36716,41.141742,-105.361760,9.000000) + [170613 12:30:47.057705] [info] BSM [RETAINED]: (ON-VGTOP,36717,41.143138,-105.361470,9.000000) + [170613 12:30:47.058086] [info] BSM [SUPPRESSED-speed]: (ON-VBTOP,36718,41.143138,-105.361470,1.000000) + [170613 12:30:47.058126] [info] BSM [RETAINED]: (ON-VGBOT,36719,41.140537,-105.362255,9.000000) + [170613 12:30:47.058147] [info] BSM [SUPPRESSED-speed]: (ON-VBBOT,36720,41.140537,-105.362255,50.000000) + [170613 12:30:47.058178] [info] BSM [RETAINED]: (ON-VG---,36721,41.411728,-110.137350,9.000000) + [170613 12:30:47.058213] [info] BSM [RETAINED]: (ON-VG-99,36722,41.411728,-110.137350,9.000000) + [170613 12:30:47.058293] [info] BSM [RETAINED]: (OFFVG---,36723,41.628687,-109.089771,9.000000) + [170613 12:30:47.058451] [info] BSM [RETAINED]: (OFFVG---,36724,41.627758,-109.091004,9.000000) + [170613 12:30:47.058494] [info] BSM [RETAINED]: (O??VG---,36725,41.627672,-109.089390,9.000000) + [170613 12:30:47.064880] [info] BSM [RETAINED]: (ON-VG---,36726,41.627467,-109.089251,9.000000) + [170613 12:30:47.064940] [info] BSM [RETAINED]: (OFFVG---,36727,43.313653,-111.799675,9.000000) + ``` + + +1. To stop the PPM and Kafka, run the following command: + + ``` + $ docker compose down + ``` + +## Kafka Integration Testing +To run kafka integration tests, run the following command: ```bash -$ ./ppm_tests +$ ./do_kafka_test.sh ``` -### Utilizing the build_and_run_unit_tests.sh script -The build_and_run_unit_test.sh script provides an easy method to build and run the PPM's unit tests. It should be noted that this script needs to have the LF end-of-line sequence for it to work. +## Test Files -#### Steps -1. Pull the project into VSCode -1. Reopen the project in a dev container -1. Open the terminal. -1. Type "sudo su" to run commands as root -1. Type ./build_and_run_unit_tests.sh to run the script +Several example JSON message test files are in the [jpo-cvdp/data](../data) directory. These files can be edited to generate +your own test cases. Each line in the file should be a well-formed BSM JSON +object. **Each message should be on a separate line in the file.** **If a JSON object cannot be parsed it is suppressed.** # See Also: Testing/Troubleshooting -More information on testing can be found in the [Testing/Troubleshooting](../README.md#Testing/Troubleshooting) section of the README. \ No newline at end of file +More information on testing can be found in the [Testing/Troubleshooting](../README.md#testingtroubleshooting) section of the README. \ No newline at end of file diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 96d57917..05cbe6fa 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -32,4 +32,4 @@ This error occurs when the PPM is unable to connect to the local broker. This ca - The PPM is targeting the wrong IP address # See Also: Testing/Troubleshooting -More information on troubleshooting can be found in the [Testing/Troubleshooting](../README.md#Testing/Troubleshooting) section of the README. \ No newline at end of file +More information on troubleshooting can be found in the [Testing/Troubleshooting](../README.md#testingtroubleshooting) section of the README. \ No newline at end of file diff --git a/sample.env b/sample.env index f23c2d0e..bb52c8db 100644 --- a/sample.env +++ b/sample.env @@ -4,13 +4,11 @@ DOCKER_HOST_IP= # An external directory that contains the necessary edges and properties files. DOCKER_SHARED_VOLUME= -# The config file to use. -# This only needs to be set when using `docker-compose.yml`. -# The `docker-compose-standalone.yml` & `docker-compose-confluent-cloud.yml` files directly set values for this. -PPM_CONFIG_FILE= +# The config file to use. This file must be in the shared volume. +PPM_CONFIG_FILE=ppmBsm.properties -# An directory that contains the fieldsToRedact.txt file needed by the RedactionPropertiesManager class. -REDACTION_PROPERTIES_PATH= +# The path to the fieldsToRedact.txt file needed by the RedactionPropertiesManager class. This file must be in the shared volume. +REDACTION_PROPERTIES_PATH=fieldsToRedact.txt # Logging related flags for file/console logging & log level PPM_LOG_TO_FILE=false diff --git a/src/bsmHandler.cpp b/src/bsmHandler.cpp index 1101a2f7..d39853ee 100644 --- a/src/bsmHandler.cpp +++ b/src/bsmHandler.cpp @@ -136,7 +136,7 @@ bool BSMHandler::isWithinEntity(BSM &bsm) const { return false; } -bool BSMHandler::process( const std::string& bsm_json ) { +bool BSMHandler::process( const std::string& message_json ) { double speed = 0.0; double latitude = 0.0; double longitude = 0.0; @@ -150,7 +150,7 @@ bool BSMHandler::process( const std::string& bsm_json ) { // create the DOM // check for errors - if (document.Parse(bsm_json.c_str()).HasParseError()) { + if (document.Parse(message_json.c_str()).HasParseError()) { result_ = ResultStatus::PARSE; return false; @@ -318,54 +318,9 @@ bool BSMHandler::process( const std::string& bsm_json ) { handleGeneralRedaction(document); // uses fieldsToRedact.txt } - else if (payload_type_str == "us.dot.its.jpo.ode.model.OdeTimPayload") { - if (!metadata.HasMember("receivedMessageDetails")) { - result_ = ResultStatus::MISSING; - - return false; - } - - rapidjson::Value& received_details = metadata["receivedMessageDetails"]; - - if (!received_details.HasMember("locationData")) { - result_ = ResultStatus::MISSING; - - return false; - } - - rapidjson::Value& location = received_details["locationData"]; - - if (!location.HasMember("latitude") || !location.HasMember("longitude") || !location.HasMember("speed")) { - result_ = ResultStatus::MISSING; - - return false; - } - - if (!location["latitude"].IsDouble() || !location["longitude"].IsDouble() || !location["speed"].IsDouble()) { - result_ = ResultStatus::OTHER; - - return false; - } - - latitude = location["latitude"].GetDouble(); - longitude = location["longitude"].GetDouble(); - speed = location["speed"].GetDouble(); - - bsm_.set_latitude(latitude); - bsm_.set_longitude(longitude); - bsm_.set_velocity(speed); - - if (is_active() && !isWithinEntity(bsm_)) { - result_ = ResultStatus::GEOPOSITION; - } - - if (is_active() && vf_.suppress(speed)) { - result_ = ResultStatus::SPEED; - } - } else { + // Unsupported payload type result_ = ResultStatus::MISSING; - return false; } diff --git a/src/tests.cpp b/src/tests.cpp index 55250ca4..2df3ce28 100644 --- a/src/tests.cpp +++ b/src/tests.cpp @@ -1309,10 +1309,6 @@ TEST_CASE( "BSMHandler JSON Id Redaction Only", "[ppm][filtering][idonly]" ) { REQUIRE ( loadTestCases( "unit-test-data/test-case.bad.speed.json", json_test_cases ) ); REQUIRE ( loadTestCases( "unit-test-data/test-case.inside.geofence.json", json_test_cases ) ); REQUIRE ( loadTestCases( "unit-test-data/test-case.outside.geofence.json", json_test_cases ) ); - REQUIRE ( loadTestCases( "unit-test-data/test-case.all.good.tims.json", json_test_cases ) ); - REQUIRE ( loadTestCases( "unit-test-data/test-case.bad.speed.tims.json", json_test_cases ) ); - REQUIRE ( loadTestCases( "unit-test-data/test-case.inside.geofence.tims.json", json_test_cases ) ); - REQUIRE ( loadTestCases( "unit-test-data/test-case.outside.geofence.tims.json", json_test_cases ) ); for ( auto& test_case : json_test_cases ) { CHECK( handler.process( test_case ) ); CHECK( handler.get_result_string() == "success" ); @@ -1351,9 +1347,6 @@ TEST_CASE( "BSMHandler JSON Speed Only Filtering", "[ppm][filtering][speedonly]" REQUIRE ( loadTestCases( "unit-test-data/test-case.inside.geofence.json", json_test_cases ) ); REQUIRE ( loadTestCases( "unit-test-data/test-case.bad.id.json", json_test_cases ) ); REQUIRE ( loadTestCases( "unit-test-data/test-case.outside.geofence.json", json_test_cases ) ); - REQUIRE ( loadTestCases( "unit-test-data/test-case.all.good.tims.json", json_test_cases ) ); - REQUIRE ( loadTestCases( "unit-test-data/test-case.inside.geofence.tims.json", json_test_cases ) ); - REQUIRE ( loadTestCases( "unit-test-data/test-case.outside.geofence.tims.json", json_test_cases ) ); for ( auto& test_case : json_test_cases ) { CHECK( handler.process( test_case ) ); CHECK( handler.get_result_string() == "success" ); @@ -1362,7 +1355,6 @@ TEST_CASE( "BSMHandler JSON Speed Only Filtering", "[ppm][filtering][speedonly]" // get rid of previous cases. json_test_cases.clear(); REQUIRE ( loadTestCases( "unit-test-data/test-case.bad.speed.json", json_test_cases ) ); - REQUIRE ( loadTestCases( "unit-test-data/test-case.bad.speed.tims.json", json_test_cases ) ); for ( auto& test_case : json_test_cases ) { CHECK_FALSE( handler.process( test_case ) ); CHECK( handler.get_result_string() == "speed" ); diff --git a/start_kafka.sh b/start_kafka.sh index 83192169..0cfdbd13 100755 --- a/start_kafka.sh +++ b/start_kafka.sh @@ -3,4 +3,4 @@ ./stop_kafka.sh # start kafka -docker-compose -f docker-compose-kafka.yml up -d \ No newline at end of file +docker compose -f docker-compose-kafka.yml up -d \ No newline at end of file diff --git a/stop_kafka.sh b/stop_kafka.sh index 5b4123b7..cf9f625b 100644 --- a/stop_kafka.sh +++ b/stop_kafka.sh @@ -1,4 +1,4 @@ #!/bin/bash # stop kafka -docker-compose -f docker-compose-kafka.yml down --remove-orphans \ No newline at end of file +docker compose -f docker-compose-kafka.yml down --remove-orphans \ No newline at end of file diff --git a/test-scripts/standalone.sh b/test-scripts/standalone.sh index df64e337..d7a681d4 100755 --- a/test-scripts/standalone.sh +++ b/test-scripts/standalone.sh @@ -2,14 +2,12 @@ # This script sets up and runs a standalone test for a PPM container. The # PPM is started in a Docker container using a specified image and waits for it to become -# ready. The script takes in three input files: ROAD_FILE, CONFIG, TEST_DATA, and a type -# argument (BSM or TIM). It checks if the input files exist and copies them to a test data +# ready. The script takes in three input files: ROAD_FILE, CONFIG, & TEST_DATA. It checks if the input files exist and copies them to a test data # directory. If the OFFSET argument is provided, it is used as the offset in the topic that # will be consumed and displayed in the output. If not, the default value of 0 is used. -# The script then produces the test data by executing either do_bsm_test.sh or do_tim_test.sh -# depending on the type argument, passing in the OFFSET value as an argument. The PPM container -# is then stopped, and the script ends. +# The script then produces the test data by executing do_bsm_test.sh, passing in the OFFSET +# value as an argument. The PPM container is then stopped, and the script ends. # This script should only be used for testing or demo purposes, as it may hang if the offsets are wrong. It also # checks if the required configuration and test data files exist before proceeding with the test. @@ -64,7 +62,7 @@ stopPPMContainer() { docker rm -f $PPM_CONTAINER_NAME > /dev/null } -USAGE="standalone.sh [MAP_FILE] [CONFIG] [TEST_FILE] [BSM | TIM] [OFFSET]" +USAGE="standalone.sh [MAP_FILE] [CONFIG] [TEST_FILE] [OFFSET]" if [ -z $1 ] || [ ! -f $1 ]; then echo "Map file: "$1" not found!" @@ -85,15 +83,9 @@ if [ -z $3 ] || [ ! -f $3 ]; then fi if [ -z $4 ]; then - echo "Must include type (BSM or TIM)!" - echo $USAGE - exit 1 -fi - -if [ -z $5 ]; then OFFSET=0 else - OFFSET=$5 + OFFSET=$4 fi mkdir -p /tmp/docker-test/data @@ -105,26 +97,15 @@ cp $1 /tmp/docker-test/data/road_file.csv # TODO replace map file line: sed -i '/TEXT_TO_BE_REPLACED/c\This line is removed by the admin.' /tmp/foo cp $2 /tmp/docker-test/data/config.properties -# Copy the data. -if [ $4 = "BSM" ]; then - cp $3 /tmp/docker-test/data/bsm_test.json -elif [ $4 = "TIM" ]; then - cp $3 /tmp/docker-test/data/tim_test.json -else - echo "Type must be BSM or TIM!" -fi +cp $3 /tmp/docker-test/data/bsm_test.json echo "**************************" -echo "Running standalone test in $PPM_CONTAINER_NAME container with "$1 $2 $3 $4 +echo "Running standalone test in $PPM_CONTAINER_NAME container with "$1 $2 $3 echo "**************************" startPPMContainer # Produce the test data. -if [ $4 = "BSM" ]; then - docker exec $PPM_CONTAINER_NAME /cvdi-stream/docker-test/do_bsm_test.sh $OFFSET -elif [ $4 = "TIM" ]; then - docker exec $PPM_CONTAINER_NAME /cvdi-stream/docker-test/do_tim_test.sh $OFFSET -fi +docker exec $PPM_CONTAINER_NAME /cvdi-stream/docker-test/do_bsm_test.sh $OFFSET stopPPMContainer \ No newline at end of file diff --git a/test-scripts/standalone_multi.sh b/test-scripts/standalone_multi.sh deleted file mode 100755 index d9616ab6..00000000 --- a/test-scripts/standalone_multi.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/bin/bash - -# This script starts two PPM containers, one for BSMs and one for TIMs, in separate Docker containers. -# It first checks if all necessary input files exist, creates necessary directories, and copies the input -# files to those directories. It then starts the PPM containers, waits for them to spin up, produces test -# data, and finally stops the containers. - -# This script should only be used for testing or demo purposes, as it may hang if the offsets are wrong. It also -# checks if the required configuration and test data files exist before proceeding with the test. - -PPM_BSM_CONTAINER_NAME=test_ppm_bsm_instance -PPM_TIM_CONTAINER_NAME=test_ppm_tim_instance -PPM_IMAGE_TAG=do-kafka-test-ppm-image -PPM_IMAGE_NAME=jpo-cvdp_ppm - -USAGE="standalone_multi.sh [MAP_FILE] [BSM_CONFIG] [TIM_CONFIG] [BSM_TEST_FILE] [TIM_TEST_FILE] [BSM_OFFSET] [TIM_OFSET]" - -startPPMContainer() { - # Start the PPM in a new container. - dockerHostIp=$DOCKER_HOST_IP - PPM_CONTAINER_NAME=$1 - data_source=$2 - ppm_container_port=$3 - - stopPPMContainer $PPM_CONTAINER_NAME - - # make sure ip can be pinged - while true; do - if ping -c 1 $dockerHostIp &> /dev/null; then - break - else - echo "Docker host ip $dockerHostIp is not pingable. Exiting." - exit 1 - fi - done - echo "Starting PPM in new container" - docker run --name $PPM_CONTAINER_NAME --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true --env PPM_LOG_LEVEL=DEBUG -v $data_source:/ppm_data -d -p $ppm_container_port':8080' $PPM_IMAGE_NAME:$PPM_IMAGE_TAG /cvdi-stream/docker-test/ppm_standalone.sh - - echo "Waiting for $PPM_CONTAINER_NAME to spin up" - # while num lines of docker logs is less than 100, sleep 1 - secondsWaited=0 - while [ $(docker logs $PPM_CONTAINER_NAME | wc -l) -lt 100 ]; do - sleep 1 - secondsWaited=$((secondsWaited+1)) - done - echo "$PPM_CONTAINER_NAME is ready after $secondsWaited seconds" - - - if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) == "0" ]; then - echo "PPM container '$PPM_CONTAINER_NAME' is not running. Exiting." - exit 1 - fi - - container_logs=$(docker logs $PPM_CONTAINER_NAME 2>&1) - if [ $(echo $container_logs | grep "Failed to make shape" | wc -l) != "0" ]; then - echo "Warning: PPM failed to make shape." - fi -} - -stopPPMContainer() { - PPM_CONTAINER_NAME=$1 - if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) != "0" ]; then - echo "Stopping existing PPM container" - docker stop $PPM_CONTAINER_NAME > /dev/null - fi - docker rm -f $PPM_CONTAINER_NAME > /dev/null -} - -if [ -z $1 ] || [ ! -f $1 ]; then - echo "Map file: "$1" not found!" - echo $USAGE - exit 1 -fi - -if [ -z $2 ] || [ ! -f $2 ]; then - echo "BSM config file: "$2" not found!" - echo $USAGE - exit 1 -fi - -if [ -z $3 ] || [ ! -f $3 ]; then - echo "TIM config file: "$3" not found!" - echo $USAGE - exit 1 -fi - -if [ -z $4 ] || [ ! -f $4 ]; then - echo "BSM test file: "$4" not found!" - echo $USAGE - exit 1 -fi - -if [ -z $5 ] || [ ! -f $5 ]; then - echo "TIM test file: "$5" not found!" - echo $USAGE - exit 1 -fi - -if [ -z $6 ]; then - BSM_OFFSET=0 -else - BSM_OFFSET=$6 -fi - -if [ -z $7 ]; then - TIM_OFFSET=0 -else - TIM_OFFSET=$7 -fi - -mkdir -p /tmp/docker-test/bsm-data -mkdir -p /tmp/docker-test/tim-data - -# Copy the road files to the docker test data. -cp $1 /tmp/docker-test/bsm-data/road_file.csv -cp $1 /tmp/docker-test/tim-data/road_file.csv - -# Copy the configs to the test data. -# TODO replace map file line: sed -i '/TEXT_TO_BE_REPLACED/c\This line is removed by the admin.' /tmp/foo -cp $2 /tmp/docker-test/bsm-data/config.properties -cp $3 /tmp/docker-test/tim-data/config.properties - -# Copy the data. -cp $4 /tmp/docker-test/bsm-data/bsm_test.json -cp $5 /tmp/docker-test/tim-data/tim_test.json - -echo "**************************" -echo "Running standalone multi PPM test with "$1 $2 $3 $4 $5 $6 $7 -echo "**************************" - -dockerHostIp=$DOCKER_HOST_IP -# Start the BSM PPM in a new container. -startPPMContainer $PPM_BSM_CONTAINER_NAME /tmp/docker-test/bsm-data 8080 - -# Start the TIM PPM in a new container. -startPPMContainer $PPM_TIM_CONTAINER_NAME /tmp/docker-test/tim-data 8081 - -# Produce the test data. -docker exec $PPM_BSM_CONTAINER_NAME /cvdi-stream/docker-test/do_bsm_test.sh $BSM_OFFSET -docker exec $PPM_TIM_CONTAINER_NAME /cvdi-stream/docker-test/do_tim_test.sh $TIM_OFFSET - -stopPPMContainer $PPM_BSM_CONTAINER_NAME -stopPPMContainer $PPM_TIM_CONTAINER_NAME diff --git a/unit-test-data/test-case.all.good.tims.json b/unit-test-data/test-case.all.good.tims.json deleted file mode 100644 index e87a51e7..00000000 --- a/unit-test-data/test-case.all.good.tims.json +++ /dev/null @@ -1,3 +0,0 @@ -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.94911, "longitude": -83.928343, "speed": 22.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.949821, "longitude": -83.936279, "speed": 22.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.951501, "longitude": -83.935851, "speed": 22.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} diff --git a/unit-test-data/test-case.bad.speed.tims.json b/unit-test-data/test-case.bad.speed.tims.json deleted file mode 100644 index 484faccb..00000000 --- a/unit-test-data/test-case.bad.speed.tims.json +++ /dev/null @@ -1,3 +0,0 @@ -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.949811, "longitude": -83.92909, "speed": 0.5}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.951084, "longitude": -83.930725, "speed": 99.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.949915, "longitude": -83.936186, "speed": 2.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} diff --git a/unit-test-data/test-case.inside.geofence.tims.json b/unit-test-data/test-case.inside.geofence.tims.json deleted file mode 100644 index c180272d..00000000 --- a/unit-test-data/test-case.inside.geofence.tims.json +++ /dev/null @@ -1,7 +0,0 @@ -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.94911, "longitude": -83.928343, "speed": 3.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.952555, "longitude": -83.932468, "speed": 4.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.949821, "longitude": -83.936279, "speed": 5.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.951501, "longitude": -83.935851, "speed": 22.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.949915, "longitude": -83.936186, "speed": 10.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.949811, "longitude": -83.92909, "speed": 34.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.951084, "longitude": -83.930725, "speed": 22.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} diff --git a/unit-test-data/test-case.outside.geofence.tims.json b/unit-test-data/test-case.outside.geofence.tims.json deleted file mode 100644 index 19bc928d..00000000 --- a/unit-test-data/test-case.outside.geofence.tims.json +++ /dev/null @@ -1,6 +0,0 @@ -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.9493, "longitude": -83.927489, "speed": 22.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.950668, "longitude": -83.931295, "speed": 22.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.962259, "longitude": -83.914569, "speed": 22.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.949271, "longitude": -83.928893, "speed": 5.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.948337, "longitude": -83.928826, "speed": 32.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3} -{"metadata": {"logFileName": "tim.uper", "odeReceivedAt": "2017-09-26T20:00:08.48Z[UTC]", "payloadType": "us.dot.its.jpo.ode.model.OdeTimPayload", "receivedMessageDetails": {"locationData": {"elevation": 101, "heading": 40, "latitude": 35.953634, "longitude": -83.931646, "speed": 5.0}, "rxFrom": 0}, "recordGeneratedAt": "2017-07-14T15:46:47.707Z[UTC]", "recordGeneratedBy": "OBU", "recordType": "receivedMsgRecord", "sanitized": false, "schemaVersion": 3, "serialId": {"bundleId": 0, "bundleSize": 1, "recordId": 0, "serialNumber": 0, "streamId": "90b148a2-4b30-46a1-9947-4084506847e8"}, "validSignature": true}, "payload": {"data": {"dataframes": [{"content": "Advisory", "crc": "0000000000000000", "durationTime": 1, "frameType": 1, "items": ["513"], "msgID": "RoadSignID", "mutcd": 5, "position": {"elevation": 917.1432, "latitude": 41.678473, "longitude": -108.782775}, "priority": 0, "regions": [{"anchorPosition": {"elevation": 2020.6969900289998, "latitude": 41.2500807, "longitude": -111.0093847}, "closedPath": false, "description": "path", "direction": "0000000000001010", "directionality": 3, "laneWidth": 7, "name": "Testing TIM", "path": {"nodes": [{"delta": "node-LL3", "nodeLat": 0.0014506, "nodeLong": 0.0031024}, {"delta": "node-LL3", "nodeLat": 0.0014568, "nodeLong": 0.0030974}, {"delta": "node-LL3", "nodeLat": 0.0014559, "nodeLong": 0.0030983}, {"delta": "node-LL3", "nodeLat": 0.0014563, "nodeLong": 0.003098}, {"delta": "node-LL3", "nodeLat": 0.0014562, "nodeLong": 0.0030982}], "scale": 0, "type": "ll"}, "regulatorID": 0, "segmentID": 33}], "sspLocationRights": 3, "sspMsgContent": 3, "sspMsgTypes": 2, "sspTimRights": 0, "startDateTime": "2017-08-02T22:25:00.000Z", "url": "null", "viewAngle": "1010101010101010"}], "index": 13, "msgCnt": 1, "packetID": 0, "timeStamp": "2016-08-03T22:25:36.297Z", "urlB": "null"}, "dataType": "us.dot.its.jpo.ode.plugin.j2735.J2735TravelerInformationMessage"}, "schemaVersion": 3}