From 970f1c2aecaaf1426d85a4dc77d353ac16f34c33 Mon Sep 17 00:00:00 2001 From: dmccoystephenson Date: Wed, 13 Apr 2022 15:23:28 -0600 Subject: [PATCH 01/51] Made some changes to resolve some issues with do_kafka_test.sh script. --- .devcontainer/devcontainer.json | 7 +++++++ Dockerfile.standalone | 3 +++ do_kafka_test.sh | 4 ++-- docker-compose.yml | 3 +++ test-scripts/standalone.sh | 12 +++++++++--- test-scripts/standalone_multi.sh | 4 ++-- 6 files changed, 26 insertions(+), 7 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 05435385..295959b1 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -19,6 +19,13 @@ "ms-vscode.cmake-tools" ], + "features": { + "docker-from-docker": { + "version": "latest", + "moby": true + } + }, + // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], diff --git a/Dockerfile.standalone b/Dockerfile.standalone index 0dcefde7..ae926212 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -24,5 +24,8 @@ ADD ./unit-test-data /cvdi-stream/unit-test-data # Do the build. RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make +# install python for testing +RUN apt-get install -y python3 + # Add test data. This changes frequently so keep it low in the file. ADD ./docker-test /cvdi-stream/docker-test diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 5d4e4f5a..c0e3b24d 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -3,11 +3,11 @@ # Wait until Kafka creates our topics. while true; do - ntopics=$(docker exec jpocvdp_kafka_1 /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 | wc -l) + ntopics=$(docker exec -it jpo-cvdp_kafka_1 /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 | wc -l) if [[ $ntopics == "4" ]]; then echo 'Found 4 topics:' - docker exec jpocvdp_kafka_1 /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 2> /dev/null + docker exec -it jpo-cvdp_kafka_1 /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 2> /dev/null break fi diff --git a/docker-compose.yml b/docker-compose.yml index efb0b638..61852b29 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,9 @@ services: ports: - "8080:8080" - "9090:9090" + environment: + DOCKER_HOST_IP: ${DOCKER_HOST_IP} + PPM_CONFIG_FILE: cdot_ppmBsm.properties depends_on: - kafka links: diff --git a/test-scripts/standalone.sh b/test-scripts/standalone.sh index 9e1cfdc1..7823776c 100755 --- a/test-scripts/standalone.sh +++ b/test-scripts/standalone.sh @@ -60,14 +60,19 @@ echo "**************************" echo "Running standalone test with "$1 $2 $3 $4 echo "**************************" -#docker stop ppm_kafka > /dev/null -#docker rm ppm_kafka > /dev/null +# echo "[log] stopping ppm_kafka" +# docker stop ppm_kafka +# echo "[log] removing ppm_kafka" +# docker rm ppm_kafka # Start the PPM in a new container. -docker run --name ppm_kafka -v /tmp/docker-test/data:/ppm_data -it --rm -p '8080:8080' -d jpocvdp_ppm:latest /cvdi-stream/docker-test/ppm_standalone.sh > /dev/null +echo "[log] starting PPM in new container" +docker run --name ppm_kafka -v /tmp/docker-test/data:/ppm_data -it --rm -p '8080:8080' jpo-cvdp_ppm:latest /cvdi-stream/docker-test/ppm_standalone.sh +echo "[log] waiting..." sleep 10 +echo "[log] producing test data" if [ $4 = "BSM" ]; then docker exec ppm_kafka /cvdi-stream/docker-test/do_bsm_test.sh $OFFSET # Produce the test data. @@ -76,4 +81,5 @@ elif [ $4 = "TIM" ]; then # Produce the test data. fi +echo "[log] stopping PPM" docker stop ppm_kafka > /dev/null diff --git a/test-scripts/standalone_multi.sh b/test-scripts/standalone_multi.sh index 404da958..71c108fc 100755 --- a/test-scripts/standalone_multi.sh +++ b/test-scripts/standalone_multi.sh @@ -69,9 +69,9 @@ echo "Running standalone multi PPM test with "$1 $2 $3 $4 $5 $6 $7 echo "**************************" # Start the BSM PPM in a new container. -docker run --name ppm_bsm_kafka -v /tmp/docker-test/bsm-data:/ppm_data -it --rm -p '8080:8080' -d jpocvdp_ppm:latest /cvdi-stream/docker-test/ppm_standalone.sh > /dev/null +docker run --name ppm_bsm_kafka -v /tmp/docker-test/bsm-data:/ppm_data -it --rm -p '8080:8080' -d jpo-cvdp_ppm:latest /cvdi-stream/docker-test/ppm_standalone.sh > /dev/null # Start the TIM PPM in a new container. -docker run --name ppm_tim_kafka -v /tmp/docker-test/tim-data:/ppm_data -it --rm -p '8081:8080' -d jpocvdp_ppm:latest /cvdi-stream/docker-test/ppm_standalone.sh > /dev/null +docker run --name ppm_tim_kafka -v /tmp/docker-test/tim-data:/ppm_data -it --rm -p '8081:8080' -d jpo-cvdp_ppm:latest /cvdi-stream/docker-test/ppm_standalone.sh > /dev/null sleep 10 From 11600d990ef1887b3040340b28d97cb56353e560 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Mon, 8 May 2023 13:36:10 -0600 Subject: [PATCH 02/51] Refactored scripts to improve clarity of execution and to minimize repetitive behavior. --- Dockerfile | 3 ++ Dockerfile-nsv | 3 ++ Dockerfile.standalone | 3 ++ do_kafka_test.sh | 58 ++++++++++++++++++++++++++--------- docker-compose-kafka.yml | 16 ++++++++++ docker-compose.yml | 3 -- docker-test/do_bsm_test.sh | 2 ++ docker-test/ppm_standalone.sh | 7 +++-- start_kafka.sh | 9 +++--- stop_kafka.sh | 4 +++ test-scripts/standalone.sh | 22 +++---------- 11 files changed, 89 insertions(+), 41 deletions(-) create mode 100644 docker-compose-kafka.yml create mode 100644 stop_kafka.sh diff --git a/Dockerfile b/Dockerfile index 600551cd..284fe674 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,9 @@ RUN apt install -y cmake # Install librdkafka. RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev +# install python for testing +RUN apt-get install -y python3 + # add the source and build files ADD CMakeLists.txt /cvdi-stream ADD ./src /cvdi-stream/src diff --git a/Dockerfile-nsv b/Dockerfile-nsv index 69cd0977..9d178d02 100644 --- a/Dockerfile-nsv +++ b/Dockerfile-nsv @@ -15,6 +15,9 @@ RUN apt install -y cmake # Install librdkafka. RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev +# install python for testing +RUN apt-get install -y python3 + # add the source and build files ADD CMakeLists.txt /cvdi-stream ADD ./src /cvdi-stream/src diff --git a/Dockerfile.standalone b/Dockerfile.standalone index ae926212..36f1372e 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -13,6 +13,9 @@ RUN apt install -y cmake # Install librdkafka. RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev +# install python for testing +RUN apt-get install -y python3 + # add the source and build files ADD CMakeLists.txt /cvdi-stream ADD ./src /cvdi-stream/src diff --git a/do_kafka_test.sh b/do_kafka_test.sh index c0e3b24d..013dc385 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -1,20 +1,55 @@ #!/bin/bash ./start_kafka.sh +KAFKA_CONTAINER_NAME=jpo-cvdp-kafka-1 + # Wait until Kafka creates our topics. while true; do - ntopics=$(docker exec -it jpo-cvdp_kafka_1 /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 | wc -l) + # if kafka container is not running, exit + if [[ $(docker ps | grep $KAFKA_CONTAINER_NAME | wc -l) == "0" ]]; then + echo "Kafka container '$KAFKA_CONTAINER_NAME' is not running. Exiting." + ./stop_kafka.sh + exit 1 + fi - if [[ $ntopics == "4" ]]; then - echo 'Found 4 topics:' - docker exec -it jpo-cvdp_kafka_1 /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 2> /dev/null + ntopics=$(docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 | wc -l) + + expected_topics=4 + if [[ $ntopics == $expected_topics ]]; then + echo '[log] found '$expected_topics' topics as expected:' + docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 2> /dev/null - break + break + elif [[ $ntopics == "0" ]]; then + echo '[log] no topics found' + else + echo '[log] found '$ntopics'/'$expected_topics' topics' fi + echo "[log] waiting for Kafka to create topics..." sleep 1 done +# build the PPM image +tag=do-kafka-test-ppm-image +PPM_IMAGE_NAME=jpo-cvdp_ppm +echo "[log] building PPM image: "$PPM_IMAGE_NAME:$tag +docker build . -t $PPM_IMAGE_NAME:$tag + +# Start the PPM in a new container. + +PPM_CONTAINER_NAME=ppm_kafka +if [[ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) != "0" ]]; then + echo "[log] stopping existing PPM container" + docker stop $PPM_CONTAINER_NAME > /dev/null +fi +echo "[log] starting PPM in new container" +docker rm -f $PPM_CONTAINER_NAME > /dev/null +dockerHostIp=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $KAFKA_CONTAINER_NAME) +echo "[log] docker host ip: '$dockerHostIp'" +docker run --name $PPM_CONTAINER_NAME -v /tmp/docker-test/data:/ppm_data -d -p '8080:8080' $PPM_IMAGE_NAME:$tag /cvdi-stream/docker-test/ppm_standalone.sh --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true + + MAP_FILE=data/I_80.edges BSM_DATA_FILE=data/I_80_test.json TIM_DATA_FILE=data/I_80_test_TIMS.json @@ -27,45 +62,40 @@ echo $TIM_DATA_FILE echo "" echo "" -sleep 1 +# sleep 1 ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c2.properties $BSM_DATA_FILE BSM 10 echo "" echo "" -sleep 1 ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c3.properties $BSM_DATA_FILE BSM 18 echo "" echo "" -sleep 1 ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c4.properties $BSM_DATA_FILE BSM 23 echo "" echo "" -sleep 1 ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c5.properties $BSM_DATA_FILE BSM 33 echo "" echo "" -sleep 1 ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c6.properties $BSM_DATA_FILE BSM 43 echo "" echo "" -sleep 1 ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c1.properties $TIM_DATA_FILE TIM 0 echo "" echo "" -sleep 1 ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c2.properties $TIM_DATA_FILE TIM 10 echo "" echo "" -sleep 1 ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c3.properties $TIM_DATA_FILE TIM 18 echo "" echo "" -sleep 1 ./test-scripts/standalone_multi.sh $MAP_FILE config/bsm-test/c6.properties config/tim-test/c3.properties $BSM_DATA_FILE $TIM_DATA_FILE 48 23 + +echo "[log] stopping PPM" +docker stop $PPM_CONTAINER_NAME > /dev/null \ No newline at end of file diff --git a/docker-compose-kafka.yml b/docker-compose-kafka.yml new file mode 100644 index 00000000..7d9a61f0 --- /dev/null +++ b/docker-compose-kafka.yml @@ -0,0 +1,16 @@ +version: '2' +services: + zookeeper: + image: wurstmeister/zookeeper + ports: + - "2181:2181" + kafka: + image: wurstmeister/kafka + ports: + - "9092:9092" + environment: + KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP} + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CREATE_TOPICS: "topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1,topic.OdeTimJson:1:1,topic.FilteredOdeTimJson:1:1" + volumes: + - /var/run/docker.sock:/var/run/docker.sock \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index a5798e29..f5c7aa11 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,9 +22,6 @@ services: ports: - "8080:8080" - "9090:9090" - environment: - DOCKER_HOST_IP: ${DOCKER_HOST_IP} - PPM_CONFIG_FILE: cdot_ppmBsm.properties depends_on: - kafka links: diff --git a/docker-test/do_bsm_test.sh b/docker-test/do_bsm_test.sh index a51fbd67..443e7b8d 100755 --- a/docker-test/do_bsm_test.sh +++ b/docker-test/do_bsm_test.sh @@ -24,5 +24,7 @@ while true; do cat tmp.out break + else + echo "[log] waiting for data..." fi done diff --git a/docker-test/ppm_standalone.sh b/docker-test/ppm_standalone.sh index 8cdd316c..7882d175 100755 --- a/docker-test/ppm_standalone.sh +++ b/docker-test/ppm_standalone.sh @@ -1,7 +1,10 @@ #!/bin/sh export LD_LIBRARY_PATH=/usr/local/lib -broker=172.17.0.1:9092 +if [[ -z "$DOCKER_HOST_IP" ]]; then + echo "DOCKER_HOST_IP is not set. Exiting." + exit 1 +fi # Start the DI tool. -/cvdi-stream-build/ppm -c /ppm_data/config.properties -o end +/cvdi-stream-build/ppm -c /ppm_data/config.properties -b ${DOCKER_HOST_IP}:9092 -o end diff --git a/start_kafka.sh b/start_kafka.sh index 1ebad0cb..83192169 100755 --- a/start_kafka.sh +++ b/start_kafka.sh @@ -1,7 +1,6 @@ #!/bin/bash -# Start the docker services. -docker-compose stop -docker-compose rm -f -v -docker-compose up --build -d -docker ps +./stop_kafka.sh + +# start kafka +docker-compose -f docker-compose-kafka.yml up -d \ No newline at end of file diff --git a/stop_kafka.sh b/stop_kafka.sh new file mode 100644 index 00000000..5b4123b7 --- /dev/null +++ b/stop_kafka.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# stop kafka +docker-compose -f docker-compose-kafka.yml down --remove-orphans \ No newline at end of file diff --git a/test-scripts/standalone.sh b/test-scripts/standalone.sh index 7823776c..b798aa95 100755 --- a/test-scripts/standalone.sh +++ b/test-scripts/standalone.sh @@ -60,26 +60,14 @@ echo "**************************" echo "Running standalone test with "$1 $2 $3 $4 echo "**************************" -# echo "[log] stopping ppm_kafka" -# docker stop ppm_kafka -# echo "[log] removing ppm_kafka" -# docker rm ppm_kafka +PPM_CONTAINER_NAME=ppm_kafka -# Start the PPM in a new container. -echo "[log] starting PPM in new container" -docker run --name ppm_kafka -v /tmp/docker-test/data:/ppm_data -it --rm -p '8080:8080' jpo-cvdp_ppm:latest /cvdi-stream/docker-test/ppm_standalone.sh - -echo "[log] waiting..." -sleep 10 - -echo "[log] producing test data" if [ $4 = "BSM" ]; then - docker exec ppm_kafka /cvdi-stream/docker-test/do_bsm_test.sh $OFFSET + echo "[log] performing bsm test in $PPM_CONTAINER_NAME container" + docker exec $PPM_CONTAINER_NAME /cvdi-stream/docker-test/do_bsm_test.sh $OFFSET # Produce the test data. elif [ $4 = "TIM" ]; then - docker exec ppm_kafka /cvdi-stream/docker-test/do_tim_test.sh $OFFSET + echo "[log] performing tim test in $PPM_CONTAINER_NAME container" + docker exec $PPM_CONTAINER_NAME /cvdi-stream/docker-test/do_tim_test.sh $OFFSET # Produce the test data. fi - -echo "[log] stopping PPM" -docker stop ppm_kafka > /dev/null From 57aebcc006faed43381b5958f68cf73d091234a0 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Mon, 8 May 2023 14:05:13 -0600 Subject: [PATCH 03/51] Organized code in do_kafka_test.sh script into methods to improve readability. --- do_kafka_test.sh | 236 ++++++++++++++++++++++++++++++----------------- 1 file changed, 149 insertions(+), 87 deletions(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 013dc385..3aafcbf2 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -1,101 +1,163 @@ #!/bin/bash -./start_kafka.sh KAFKA_CONTAINER_NAME=jpo-cvdp-kafka-1 - -# Wait until Kafka creates our topics. -while true; do - # if kafka container is not running, exit - if [[ $(docker ps | grep $KAFKA_CONTAINER_NAME | wc -l) == "0" ]]; then - echo "Kafka container '$KAFKA_CONTAINER_NAME' is not running. Exiting." - ./stop_kafka.sh - exit 1 - fi - - ntopics=$(docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 | wc -l) - - expected_topics=4 - if [[ $ntopics == $expected_topics ]]; then - echo '[log] found '$expected_topics' topics as expected:' - docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 2> /dev/null - - break - elif [[ $ntopics == "0" ]]; then - echo '[log] no topics found' - else - echo '[log] found '$ntopics'/'$expected_topics' topics' - fi - - echo "[log] waiting for Kafka to create topics..." - sleep 1 -done - -# build the PPM image -tag=do-kafka-test-ppm-image -PPM_IMAGE_NAME=jpo-cvdp_ppm -echo "[log] building PPM image: "$PPM_IMAGE_NAME:$tag -docker build . -t $PPM_IMAGE_NAME:$tag - -# Start the PPM in a new container. - -PPM_CONTAINER_NAME=ppm_kafka -if [[ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) != "0" ]]; then - echo "[log] stopping existing PPM container" - docker stop $PPM_CONTAINER_NAME > /dev/null -fi -echo "[log] starting PPM in new container" -docker rm -f $PPM_CONTAINER_NAME > /dev/null -dockerHostIp=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $KAFKA_CONTAINER_NAME) -echo "[log] docker host ip: '$dockerHostIp'" -docker run --name $PPM_CONTAINER_NAME -v /tmp/docker-test/data:/ppm_data -d -p '8080:8080' $PPM_IMAGE_NAME:$tag /cvdi-stream/docker-test/ppm_standalone.sh --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true - - MAP_FILE=data/I_80.edges BSM_DATA_FILE=data/I_80_test.json TIM_DATA_FILE=data/I_80_test_TIMS.json -echo $MAP_FILE -echo $BSM_DATA_FILE -echo $TIM_DATA_FILE - -./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c1.properties $BSM_DATA_FILE BSM 0 -echo "" -echo "" - -# sleep 1 -./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c2.properties $BSM_DATA_FILE BSM 10 -echo "" -echo "" - -./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c3.properties $BSM_DATA_FILE BSM 18 -echo "" -echo "" - -./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c4.properties $BSM_DATA_FILE BSM 23 -echo "" -echo "" +setup() { + echo "[log] setting up test environment" + ./start_kafka.sh +} + +waitForKafkaToCreateTopics() { + # Wait until Kafka creates our topics. + while true; do + # if kafka container is not running, exit + if [[ $(docker ps | grep $KAFKA_CONTAINER_NAME | wc -l) == "0" ]]; then + echo "Kafka container '$KAFKA_CONTAINER_NAME' is not running. Exiting." + ./stop_kafka.sh + exit 1 + fi + + ntopics=$(docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 | wc -l) + + expected_topics=4 + if [[ $ntopics == $expected_topics ]]; then + echo '[log] found '$expected_topics' topics as expected:' + docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 2> /dev/null + + break + elif [[ $ntopics == "0" ]]; then + echo '[log] no topics found' + else + echo '[log] found '$ntopics'/'$expected_topics' topics' + fi + + echo "[log] waiting for Kafka to create topics..." + sleep 1 + done +} + +buildPPMImage() { + # build the PPM image + tag=do-kafka-test-ppm-image + PPM_IMAGE_NAME=jpo-cvdp_ppm + echo "[log] building PPM image: "$PPM_IMAGE_NAME:$tag + docker build . -t $PPM_IMAGE_NAME:$tag +} + + +startPPMContainer() { + # Start the PPM in a new container. + PPM_CONTAINER_NAME=ppm_kafka + if [[ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) != "0" ]]; then + echo "[log] stopping existing PPM container" + docker stop $PPM_CONTAINER_NAME > /dev/null + fi + echo "[log] starting PPM in new container" + docker rm -f $PPM_CONTAINER_NAME > /dev/null + dockerHostIp=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $KAFKA_CONTAINER_NAME) + echo "[log] docker host ip: '$dockerHostIp'" + docker run --name $PPM_CONTAINER_NAME -v /tmp/docker-test/data:/ppm_data -d -p '8080:8080' $PPM_IMAGE_NAME:$tag /cvdi-stream/docker-test/ppm_standalone.sh --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true + + # wait until container spins up + while true; do + if [[ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) == "0" ]]; then + echo "PPM container '$PPM_CONTAINER_NAME' is not running. Exiting." + exit 1 + fi + + nlines=$(docker logs $PPM_CONTAINER_NAME 2>&1 | wc -l) + if [[ $nlines == "0" ]]; then + echo "[log] waiting for PPM to start..." + else + echo "--- PPM LOGS ---" + docker logs $PPM_CONTAINER_NAME + echo "----------------" + break + fi + + sleep 1 + done +} + +run_tests() { + echo $MAP_FILE + echo $BSM_DATA_FILE + echo $TIM_DATA_FILE + + ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c1.properties $BSM_DATA_FILE BSM 0 + echo "" + echo "" + + # sleep 1 + ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c2.properties $BSM_DATA_FILE BSM 10 + echo "" + echo "" + + ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c3.properties $BSM_DATA_FILE BSM 18 + echo "" + echo "" + + ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c4.properties $BSM_DATA_FILE BSM 23 + echo "" + echo "" + + ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c5.properties $BSM_DATA_FILE BSM 33 + echo "" + echo "" + + ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c6.properties $BSM_DATA_FILE BSM 43 + echo "" + echo "" + + ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c1.properties $TIM_DATA_FILE TIM 0 + echo "" + echo "" + + ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c2.properties $TIM_DATA_FILE TIM 10 + echo "" + echo "" + + ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c3.properties $TIM_DATA_FILE TIM 18 + echo "" + echo "" + + ./test-scripts/standalone_multi.sh $MAP_FILE config/bsm-test/c6.properties config/tim-test/c3.properties $BSM_DATA_FILE $TIM_DATA_FILE 48 23 +} + +cleanup() { + echo "[log] stopping PPM" + docker stop $PPM_CONTAINER_NAME > /dev/null + echo "[log] stopping Kafka" + ./stop_kafka.sh +} -./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c5.properties $BSM_DATA_FILE BSM 33 -echo "" -echo "" +run() { + echo "" + echo "Step 1/6: Set up test environment" + setup -./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c6.properties $BSM_DATA_FILE BSM 43 -echo "" -echo "" + echo "" + echo "Step 2/6: Wait for Kafka to create topics" + waitForKafkaToCreateTopics -./test-scripts/standalone.sh $MAP_FILE config/tim-test/c1.properties $TIM_DATA_FILE TIM 0 -echo "" -echo "" + echo "" + echo "Step 3/6: Build PPM image" + buildPPMImage -./test-scripts/standalone.sh $MAP_FILE config/tim-test/c2.properties $TIM_DATA_FILE TIM 10 -echo "" -echo "" + echo "" + echo "Step 4/6: Start PPM container" + startPPMContainer -./test-scripts/standalone.sh $MAP_FILE config/tim-test/c3.properties $TIM_DATA_FILE TIM 18 -echo "" -echo "" + echo "" + echo "Step 5/6: Run tests" + run_tests -./test-scripts/standalone_multi.sh $MAP_FILE config/bsm-test/c6.properties config/tim-test/c3.properties $BSM_DATA_FILE $TIM_DATA_FILE 48 23 + echo "" + echo "Step 6/6: Cleanup" + cleanup +} -echo "[log] stopping PPM" -docker stop $PPM_CONTAINER_NAME > /dev/null \ No newline at end of file +run From 24960242cffe1abdff4afae3a0e5e4439b03e300 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Mon, 8 May 2023 15:14:28 -0600 Subject: [PATCH 04/51] Made the DOCKER_HOST_IP env var required for the do_kafka_test.sh script. --- do_kafka_test.sh | 36 +++++++++++++++++++++++++---------- docker-test/ppm_standalone.sh | 2 +- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 3aafcbf2..ed224183 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -6,7 +6,11 @@ BSM_DATA_FILE=data/I_80_test.json TIM_DATA_FILE=data/I_80_test_TIMS.json setup() { - echo "[log] setting up test environment" + # exit if DOCKER_HOST_IP is not set + if [ -z $DOCKER_HOST_IP ]; then + echo "[log] DOCKER_HOST_IP is not set. Exiting." + exit 1 + fi ./start_kafka.sh } @@ -14,7 +18,7 @@ waitForKafkaToCreateTopics() { # Wait until Kafka creates our topics. while true; do # if kafka container is not running, exit - if [[ $(docker ps | grep $KAFKA_CONTAINER_NAME | wc -l) == "0" ]]; then + if [ $(docker ps | grep $KAFKA_CONTAINER_NAME | wc -l) == "0" ]; then echo "Kafka container '$KAFKA_CONTAINER_NAME' is not running. Exiting." ./stop_kafka.sh exit 1 @@ -23,12 +27,12 @@ waitForKafkaToCreateTopics() { ntopics=$(docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 | wc -l) expected_topics=4 - if [[ $ntopics == $expected_topics ]]; then + if [ $ntopics == $expected_topics ]; then echo '[log] found '$expected_topics' topics as expected:' docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 2> /dev/null break - elif [[ $ntopics == "0" ]]; then + elif [ $ntopics == "0" ]; then echo '[log] no topics found' else echo '[log] found '$ntopics'/'$expected_topics' topics' @@ -51,25 +55,37 @@ buildPPMImage() { startPPMContainer() { # Start the PPM in a new container. PPM_CONTAINER_NAME=ppm_kafka - if [[ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) != "0" ]]; then + if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) != "0" ]; then echo "[log] stopping existing PPM container" docker stop $PPM_CONTAINER_NAME > /dev/null fi - echo "[log] starting PPM in new container" docker rm -f $PPM_CONTAINER_NAME > /dev/null - dockerHostIp=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $KAFKA_CONTAINER_NAME) + + dockerHostIp=$DOCKER_HOST_IP echo "[log] docker host ip: '$dockerHostIp'" - docker run --name $PPM_CONTAINER_NAME -v /tmp/docker-test/data:/ppm_data -d -p '8080:8080' $PPM_IMAGE_NAME:$tag /cvdi-stream/docker-test/ppm_standalone.sh --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true + + # make sure ip can be pinged + while true; do + if ping -c 1 $dockerHostIp &> /dev/null; then + echo "[log] docker host ip is pingable" + break + else + echo "[log] docker host ip is not pingable. Exiting." + exit 1 + fi + done + echo "[log] starting PPM in new container" + docker run --name $PPM_CONTAINER_NAME --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true -v /tmp/docker-test/data:/ppm_data -d -p '8080:8080' $PPM_IMAGE_NAME:$tag /cvdi-stream/docker-test/ppm_standalone.sh # wait until container spins up while true; do - if [[ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) == "0" ]]; then + if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) == "0" ]; then echo "PPM container '$PPM_CONTAINER_NAME' is not running. Exiting." exit 1 fi nlines=$(docker logs $PPM_CONTAINER_NAME 2>&1 | wc -l) - if [[ $nlines == "0" ]]; then + if [ $nlines == "0" ]; then echo "[log] waiting for PPM to start..." else echo "--- PPM LOGS ---" diff --git a/docker-test/ppm_standalone.sh b/docker-test/ppm_standalone.sh index 7882d175..43aba121 100755 --- a/docker-test/ppm_standalone.sh +++ b/docker-test/ppm_standalone.sh @@ -1,7 +1,7 @@ #!/bin/sh export LD_LIBRARY_PATH=/usr/local/lib -if [[ -z "$DOCKER_HOST_IP" ]]; then +if [ -z "$DOCKER_HOST_IP" ]; then echo "DOCKER_HOST_IP is not set. Exiting." exit 1 fi From 7d6d8c8216810e8e1f906c261b5dc7a1c7359b31 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Mon, 8 May 2023 15:55:55 -0600 Subject: [PATCH 05/51] Modified topic creation check in do_kafka_test.sh script. --- do_kafka_test.sh | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index ed224183..529b748b 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -24,18 +24,26 @@ waitForKafkaToCreateTopics() { exit 1 fi - ntopics=$(docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 | wc -l) - - expected_topics=4 - if [ $ntopics == $expected_topics ]; then - echo '[log] found '$expected_topics' topics as expected:' - docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 2> /dev/null - - break - elif [ $ntopics == "0" ]; then - echo '[log] no topics found' + ltopics=$(docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1) + + # required topics: + # - topic.FilteredOdeBsmJson + # - topic.FilteredOdeTimJson + # - topic.OdeBsmJson + # - topic.OdeTimJson + + # use greps to check ltopics for required topics + if [ $(echo $ltopics | grep "topic.FilteredOdeBsmJson" | wc -l) == "0" ]; then + echo "[log] Kafka has not created topic 'topic.FilteredOdeBsmJson'" + elif [ $(echo $ltopics | grep "topic.FilteredOdeTimJson" | wc -l) == "0" ]; then + echo "[log] Kafka has not created topic 'topic.FilteredOdeTimJson'" + elif [ $(echo $ltopics | grep "topic.OdeBsmJson" | wc -l) == "0" ]; then + echo "[log] Kafka has not created topic 'topic.OdeBsmJson'" + elif [ $(echo $ltopics | grep "topic.OdeTimJson" | wc -l) == "0" ]; then + echo "[log] Kafka has not created topic 'topic.OdeTimJson'" else - echo '[log] found '$ntopics'/'$expected_topics' topics' + echo "[log] Kafka has created all required topics" + break fi echo "[log] waiting for Kafka to create topics..." From 19af7922e5cd1a37e0a811a3fa4d3c48833baf8a Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Tue, 9 May 2023 12:08:08 -0600 Subject: [PATCH 06/51] Moved the responsibility of starting/stopping the PPM container back to the `standalone.sh` script. --- do_kafka_test.sh | 94 +++++++++++++------------------------- docker-test/do_bsm_test.sh | 11 ++++- docker-test/do_tim_test.sh | 13 +++++- test-scripts/standalone.sh | 54 ++++++++++++++++++++-- 4 files changed, 104 insertions(+), 68 deletions(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 529b748b..85a012c8 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -4,6 +4,9 @@ KAFKA_CONTAINER_NAME=jpo-cvdp-kafka-1 MAP_FILE=data/I_80.edges BSM_DATA_FILE=data/I_80_test.json TIM_DATA_FILE=data/I_80_test_TIMS.json +PPM_CONTAINER_NAME=ppm_kafka +PPM_IMAGE_TAG=do-kafka-test-ppm-image +PPM_IMAGE_NAME=jpo-cvdp_ppm setup() { # exit if DOCKER_HOST_IP is not set @@ -11,6 +14,13 @@ setup() { echo "[log] DOCKER_HOST_IP is not set. Exiting." exit 1 fi + + if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) != "0" ]; then + echo "[log] stopping existing PPM container" + docker stop $PPM_CONTAINER_NAME > /dev/null + fi + docker rm -f $PPM_CONTAINER_NAME > /dev/null + ./start_kafka.sh } @@ -53,134 +63,92 @@ waitForKafkaToCreateTopics() { buildPPMImage() { # build the PPM image - tag=do-kafka-test-ppm-image - PPM_IMAGE_NAME=jpo-cvdp_ppm - echo "[log] building PPM image: "$PPM_IMAGE_NAME:$tag - docker build . -t $PPM_IMAGE_NAME:$tag -} - - -startPPMContainer() { - # Start the PPM in a new container. - PPM_CONTAINER_NAME=ppm_kafka - if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) != "0" ]; then - echo "[log] stopping existing PPM container" - docker stop $PPM_CONTAINER_NAME > /dev/null - fi - docker rm -f $PPM_CONTAINER_NAME > /dev/null - - dockerHostIp=$DOCKER_HOST_IP - echo "[log] docker host ip: '$dockerHostIp'" - - # make sure ip can be pinged - while true; do - if ping -c 1 $dockerHostIp &> /dev/null; then - echo "[log] docker host ip is pingable" - break - else - echo "[log] docker host ip is not pingable. Exiting." - exit 1 - fi - done - echo "[log] starting PPM in new container" - docker run --name $PPM_CONTAINER_NAME --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true -v /tmp/docker-test/data:/ppm_data -d -p '8080:8080' $PPM_IMAGE_NAME:$tag /cvdi-stream/docker-test/ppm_standalone.sh - - # wait until container spins up - while true; do - if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) == "0" ]; then - echo "PPM container '$PPM_CONTAINER_NAME' is not running. Exiting." - exit 1 - fi - - nlines=$(docker logs $PPM_CONTAINER_NAME 2>&1 | wc -l) - if [ $nlines == "0" ]; then - echo "[log] waiting for PPM to start..." - else - echo "--- PPM LOGS ---" - docker logs $PPM_CONTAINER_NAME - echo "----------------" - break - fi - - sleep 1 - done + echo "[log] building PPM image: "$PPM_IMAGE_NAME:$PPM_IMAGE_TAG + docker build . -t $PPM_IMAGE_NAME:$PPM_IMAGE_TAG } run_tests() { + echo "--- File Being Used ---" echo $MAP_FILE echo $BSM_DATA_FILE echo $TIM_DATA_FILE + echo "-----------------" + numberOfTests=10 + echo "Test 1/$numberOfTests" ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c1.properties $BSM_DATA_FILE BSM 0 echo "" echo "" - # sleep 1 + echo "Test 2/$numberOfTests" ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c2.properties $BSM_DATA_FILE BSM 10 echo "" echo "" + echo "Test 3/$numberOfTests" ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c3.properties $BSM_DATA_FILE BSM 18 echo "" echo "" + echo "Test 4/$numberOfTests" ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c4.properties $BSM_DATA_FILE BSM 23 echo "" echo "" + echo "Test 5/$numberOfTests" ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c5.properties $BSM_DATA_FILE BSM 33 echo "" echo "" + echo "Test 6/$numberOfTests" ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c6.properties $BSM_DATA_FILE BSM 43 echo "" echo "" + echo "Test 7/$numberOfTests" ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c1.properties $TIM_DATA_FILE TIM 0 echo "" echo "" + echo "Test 8/$numberOfTests" ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c2.properties $TIM_DATA_FILE TIM 10 echo "" echo "" + echo "Test 9/$numberOfTests" ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c3.properties $TIM_DATA_FILE TIM 18 echo "" echo "" + echo "Test 10/$numberOfTests" ./test-scripts/standalone_multi.sh $MAP_FILE config/bsm-test/c6.properties config/tim-test/c3.properties $BSM_DATA_FILE $TIM_DATA_FILE 48 23 } cleanup() { - echo "[log] stopping PPM" - docker stop $PPM_CONTAINER_NAME > /dev/null echo "[log] stopping Kafka" ./stop_kafka.sh } run() { + numberOfSteps=5 echo "" - echo "Step 1/6: Set up test environment" + echo "Step 1/$numberOfSteps: Set up test environment" setup echo "" - echo "Step 2/6: Wait for Kafka to create topics" + echo "Step 2/$numberOfSteps: Wait for Kafka to create topics" waitForKafkaToCreateTopics echo "" - echo "Step 3/6: Build PPM image" + echo "Step 3/$numberOfSteps: Build PPM image" buildPPMImage echo "" - echo "Step 4/6: Start PPM container" - startPPMContainer - - echo "" - echo "Step 5/6: Run tests" + echo "Step 4/$numberOfSteps: Run tests" run_tests echo "" - echo "Step 6/6: Cleanup" + echo "Step 5/$numberOfSteps: Cleanup" cleanup } diff --git a/docker-test/do_bsm_test.sh b/docker-test/do_bsm_test.sh index 443e7b8d..ee5f53a0 100755 --- a/docker-test/do_bsm_test.sh +++ b/docker-test/do_bsm_test.sh @@ -1,7 +1,7 @@ #!/bin/bash export LD_LIBRARY_PATH=/usr/local/lib -broker=172.17.0.1:9092 +broker=$DOCKER_HOST_IP:9092 echo "**************************" echo "Producing Raw BSMs..." @@ -15,7 +15,11 @@ echo "**************************" echo "Consuming Filtered BSMs at offset "$offset "..." echo "**************************" +attempts=0 +max_attempts=5 while true; do + attempts=$((attempts+1)) + /cvdi-stream-build/kafka-test/kafka_tool -C -b $broker -p 0 -t topic.FilteredOdeBsmJson -e -o $offset 2> con.err | /cvdi-stream/docker-test/test_out.py > tmp.out lines=$(cat tmp.out | wc -l) @@ -25,6 +29,11 @@ while true; do break else + if [[ $attempts > $max_attempts ]]; then + echo "[log] no data received after $max_attempts attempts, exiting..." + exit 1 + fi echo "[log] waiting for data..." fi done +echo "number of attempts taken: $attempts" \ No newline at end of file diff --git a/docker-test/do_tim_test.sh b/docker-test/do_tim_test.sh index d87c3666..a5ac8349 100755 --- a/docker-test/do_tim_test.sh +++ b/docker-test/do_tim_test.sh @@ -1,7 +1,7 @@ #!/bin/bash export LD_LIBRARY_PATH=/usr/local/lib -broker=172.17.0.1:9092 +broker=$DOCKER_HOST_IP:9092 echo "**************************" echo "Producing Raw TIMs..." @@ -15,7 +15,11 @@ echo "**************************" echo "Consuming Filtered TIMs at offset "$offset "..." echo "**************************" +attempts=0 +max_attempts=5 while true; do + attempts=$((attempts+1)) + /cvdi-stream-build/kafka-test/kafka_tool -C -b $broker -p 0 -t topic.FilteredOdeTimJson -e -o $offset 2> con.err | /cvdi-stream/docker-test/test_out.py > tmp.out lines=$(cat tmp.out | wc -l) @@ -24,5 +28,12 @@ while true; do cat tmp.out break + else + if [[ $attempts > $max_attempts ]]; then + echo "[log] no data received after $max_attempts attempts, exiting..." + exit 1 + fi + echo "[log] waiting for data..." fi done +echo "number of attempts taken: $attempts" \ No newline at end of file diff --git a/test-scripts/standalone.sh b/test-scripts/standalone.sh index b798aa95..8b144966 100755 --- a/test-scripts/standalone.sh +++ b/test-scripts/standalone.sh @@ -6,6 +6,54 @@ # There are three input files: ROAD_FILE, CONFIG, TEST_DATA. # Offset is the offset in the topic that will be consumed and displayed in the # output + +PPM_CONTAINER_NAME=ppm_kafka +PPM_IMAGE_TAG=do-kafka-test-ppm-image +PPM_IMAGE_NAME=jpo-cvdp_ppm + +startPPMContainer() { + # Start the PPM in a new container. + dockerHostIp=$DOCKER_HOST_IP + echo "[log] docker host ip: '$dockerHostIp'" + + # make sure ip can be pinged + while true; do + if ping -c 1 $dockerHostIp &> /dev/null; then + echo "[log] docker host ip is pingable" + break + else + echo "[log] docker host ip is not pingable. Exiting." + exit 1 + fi + done + echo "[log] starting PPM in new container" + docker run --name $PPM_CONTAINER_NAME --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true -v /tmp/docker-test/data:/ppm_data -d -p '8080:8080' $PPM_IMAGE_NAME:$PPM_IMAGE_TAG /cvdi-stream/docker-test/ppm_standalone.sh + + # wait until container spins up + while true; do + if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) == "0" ]; then + echo "PPM container '$PPM_CONTAINER_NAME' is not running. Exiting." + exit 1 + fi + break + sleep 1 + echo "[log] waiting for PPM to start..." + done + + container_logs=$(docker logs $PPM_CONTAINER_NAME 2>&1) + if [ $(echo $container_logs | grep "Failed to make shape" | wc -l) != "0" ]; then + echo "[log] Warning: PPM failed to make shape." + fi +} + +stopPPMContainer() { + if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) != "0" ]; then + echo "[log] stopping existing PPM container" + docker stop $PPM_CONTAINER_NAME > /dev/null + fi + docker rm -f $PPM_CONTAINER_NAME > /dev/null +} + USAGE="standalone.sh [MAP_FILE] [CONFIG] [TEST_FILE] [BSM | TIM] [OFFSET]" if [ -z $1 ] || [ ! -f $1 ]; then @@ -60,14 +108,14 @@ echo "**************************" echo "Running standalone test with "$1 $2 $3 $4 echo "**************************" -PPM_CONTAINER_NAME=ppm_kafka +startPPMContainer if [ $4 = "BSM" ]; then echo "[log] performing bsm test in $PPM_CONTAINER_NAME container" docker exec $PPM_CONTAINER_NAME /cvdi-stream/docker-test/do_bsm_test.sh $OFFSET - # Produce the test data. elif [ $4 = "TIM" ]; then echo "[log] performing tim test in $PPM_CONTAINER_NAME container" docker exec $PPM_CONTAINER_NAME /cvdi-stream/docker-test/do_tim_test.sh $OFFSET - # Produce the test data. fi + +stopPPMContainer \ No newline at end of file From c6bb6760370ef33451c4ec29ce5091b0e4a7e8d6 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Tue, 9 May 2023 13:18:47 -0600 Subject: [PATCH 07/51] Modified & added color to echo messages. --- do_kafka_test.sh | 67 +++++++++++++++++--------------------- docker-test/do_bsm_test.sh | 14 ++++++-- docker-test/do_tim_test.sh | 14 ++++++-- test-scripts/standalone.sh | 24 +++++++------- 4 files changed, 65 insertions(+), 54 deletions(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 85a012c8..1bf16440 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -1,5 +1,9 @@ #!/bin/bash +CYAN='\033[0;36m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + KAFKA_CONTAINER_NAME=jpo-cvdp-kafka-1 MAP_FILE=data/I_80.edges BSM_DATA_FILE=data/I_80_test.json @@ -9,14 +13,13 @@ PPM_IMAGE_TAG=do-kafka-test-ppm-image PPM_IMAGE_NAME=jpo-cvdp_ppm setup() { - # exit if DOCKER_HOST_IP is not set if [ -z $DOCKER_HOST_IP ]; then - echo "[log] DOCKER_HOST_IP is not set. Exiting." + echo "DOCKER_HOST_IP is not set. Exiting." exit 1 fi if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) != "0" ]; then - echo "[log] stopping existing PPM container" + echo "Stopping existing PPM container" docker stop $PPM_CONTAINER_NAME > /dev/null fi docker rm -f $PPM_CONTAINER_NAME > /dev/null @@ -25,9 +28,7 @@ setup() { } waitForKafkaToCreateTopics() { - # Wait until Kafka creates our topics. while true; do - # if kafka container is not running, exit if [ $(docker ps | grep $KAFKA_CONTAINER_NAME | wc -l) == "0" ]; then echo "Kafka container '$KAFKA_CONTAINER_NAME' is not running. Exiting." ./stop_kafka.sh @@ -35,35 +36,27 @@ waitForKafkaToCreateTopics() { fi ltopics=$(docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1) - - # required topics: - # - topic.FilteredOdeBsmJson - # - topic.FilteredOdeTimJson - # - topic.OdeBsmJson - # - topic.OdeTimJson - - # use greps to check ltopics for required topics + allTopicsCreated=true if [ $(echo $ltopics | grep "topic.FilteredOdeBsmJson" | wc -l) == "0" ]; then - echo "[log] Kafka has not created topic 'topic.FilteredOdeBsmJson'" + allTopicsCreated=false elif [ $(echo $ltopics | grep "topic.FilteredOdeTimJson" | wc -l) == "0" ]; then - echo "[log] Kafka has not created topic 'topic.FilteredOdeTimJson'" + allTopicsCreated=false elif [ $(echo $ltopics | grep "topic.OdeBsmJson" | wc -l) == "0" ]; then - echo "[log] Kafka has not created topic 'topic.OdeBsmJson'" + allTopicsCreated=false elif [ $(echo $ltopics | grep "topic.OdeTimJson" | wc -l) == "0" ]; then - echo "[log] Kafka has not created topic 'topic.OdeTimJson'" - else - echo "[log] Kafka has created all required topics" + allTopicsCreated=false + fi + + if [ $allTopicsCreated == true ]; then + echo "Kafka has created all required topics" break fi - echo "[log] waiting for Kafka to create topics..." sleep 1 done } buildPPMImage() { - # build the PPM image - echo "[log] building PPM image: "$PPM_IMAGE_NAME:$PPM_IMAGE_TAG docker build . -t $PPM_IMAGE_NAME:$PPM_IMAGE_TAG } @@ -75,52 +68,52 @@ run_tests() { echo "-----------------" numberOfTests=10 - echo "Test 1/$numberOfTests" + echo -e $YELLOW"Test 1/$numberOfTests"$NC ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c1.properties $BSM_DATA_FILE BSM 0 echo "" echo "" - echo "Test 2/$numberOfTests" + echo -e $YELLOW"Test 2/$numberOfTests"$NC ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c2.properties $BSM_DATA_FILE BSM 10 echo "" echo "" - echo "Test 3/$numberOfTests" + echo -e $YELLOW"Test 3/$numberOfTests"$NC ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c3.properties $BSM_DATA_FILE BSM 18 echo "" echo "" - echo "Test 4/$numberOfTests" + echo -e $YELLOW"Test 4/$numberOfTests"$NC ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c4.properties $BSM_DATA_FILE BSM 23 echo "" echo "" - echo "Test 5/$numberOfTests" + echo -e $YELLOW"Test 5/$numberOfTests"$NC ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c5.properties $BSM_DATA_FILE BSM 33 echo "" echo "" - echo "Test 6/$numberOfTests" + echo -e $YELLOW"Test 6/$numberOfTests"$NC ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c6.properties $BSM_DATA_FILE BSM 43 echo "" echo "" - echo "Test 7/$numberOfTests" + echo -e $YELLOW"Test 7/$numberOfTests"$NC ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c1.properties $TIM_DATA_FILE TIM 0 echo "" echo "" - echo "Test 8/$numberOfTests" + echo -e $YELLOW"Test 8/$numberOfTests"$NC ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c2.properties $TIM_DATA_FILE TIM 10 echo "" echo "" - echo "Test 9/$numberOfTests" + echo -e $YELLOW"Test 9/$numberOfTests"$NC ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c3.properties $TIM_DATA_FILE TIM 18 echo "" echo "" - echo "Test 10/$numberOfTests" + echo -e $YELLOW"Test 10/$numberOfTests"$NC ./test-scripts/standalone_multi.sh $MAP_FILE config/bsm-test/c6.properties config/tim-test/c3.properties $BSM_DATA_FILE $TIM_DATA_FILE 48 23 } @@ -132,23 +125,23 @@ cleanup() { run() { numberOfSteps=5 echo "" - echo "Step 1/$numberOfSteps: Set up test environment" + echo -e $CYAN"Step 1/$numberOfSteps: Set up test environment"$NC setup echo "" - echo "Step 2/$numberOfSteps: Wait for Kafka to create topics" + echo -e $CYAN"Step 2/$numberOfSteps: Wait for Kafka to create topics"$NC waitForKafkaToCreateTopics echo "" - echo "Step 3/$numberOfSteps: Build PPM image" + echo -e $CYAN"Step 3/$numberOfSteps: Build PPM image"$NC buildPPMImage echo "" - echo "Step 4/$numberOfSteps: Run tests" + echo -e $CYAN"Step 4/$numberOfSteps: Run tests"$NC run_tests echo "" - echo "Step 5/$numberOfSteps: Cleanup" + echo -e $CYAN"Step 5/$numberOfSteps: Cleanup"$NC cleanup } diff --git a/docker-test/do_bsm_test.sh b/docker-test/do_bsm_test.sh index ee5f53a0..fb399bac 100755 --- a/docker-test/do_bsm_test.sh +++ b/docker-test/do_bsm_test.sh @@ -1,6 +1,10 @@ #!/bin/bash export LD_LIBRARY_PATH=/usr/local/lib +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' # No Color + broker=$DOCKER_HOST_IP:9092 echo "**************************" @@ -30,10 +34,14 @@ while true; do break else if [[ $attempts > $max_attempts ]]; then - echo "[log] no data received after $max_attempts attempts, exiting..." + echo "No data received after $max_attempts attempts. Exiting..." + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo -e $RED"TEST FAILED!"$NC + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" exit 1 fi - echo "[log] waiting for data..." fi done -echo "number of attempts taken: $attempts" \ No newline at end of file +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo -e $GREEN"TEST PASSED!"$NC +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" \ No newline at end of file diff --git a/docker-test/do_tim_test.sh b/docker-test/do_tim_test.sh index a5ac8349..ca44d2d7 100755 --- a/docker-test/do_tim_test.sh +++ b/docker-test/do_tim_test.sh @@ -1,6 +1,10 @@ #!/bin/bash export LD_LIBRARY_PATH=/usr/local/lib +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' # No Color + broker=$DOCKER_HOST_IP:9092 echo "**************************" @@ -30,10 +34,14 @@ while true; do break else if [[ $attempts > $max_attempts ]]; then - echo "[log] no data received after $max_attempts attempts, exiting..." + echo "No data received after $max_attempts attempts. Exiting..." + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo -e $RED"TEST FAILED!"$NC + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" exit 1 fi - echo "[log] waiting for data..." fi done -echo "number of attempts taken: $attempts" \ No newline at end of file +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo -e $GREEN"TEST PASSED!"$NC +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~" \ No newline at end of file diff --git a/test-scripts/standalone.sh b/test-scripts/standalone.sh index 8b144966..d7b784f9 100755 --- a/test-scripts/standalone.sh +++ b/test-scripts/standalone.sh @@ -14,19 +14,17 @@ PPM_IMAGE_NAME=jpo-cvdp_ppm startPPMContainer() { # Start the PPM in a new container. dockerHostIp=$DOCKER_HOST_IP - echo "[log] docker host ip: '$dockerHostIp'" # make sure ip can be pinged while true; do if ping -c 1 $dockerHostIp &> /dev/null; then - echo "[log] docker host ip is pingable" break else - echo "[log] docker host ip is not pingable. Exiting." + echo "Docker host ip $dockerHostIp is not pingable. Exiting." exit 1 fi done - echo "[log] starting PPM in new container" + echo "Starting PPM in new container" docker run --name $PPM_CONTAINER_NAME --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true -v /tmp/docker-test/data:/ppm_data -d -p '8080:8080' $PPM_IMAGE_NAME:$PPM_IMAGE_TAG /cvdi-stream/docker-test/ppm_standalone.sh # wait until container spins up @@ -35,20 +33,26 @@ startPPMContainer() { echo "PPM container '$PPM_CONTAINER_NAME' is not running. Exiting." exit 1 fi - break + + container_logs=$(docker logs $PPM_CONTAINER_NAME 2>&1) + indicator="BSMHandler::BSMHandler(): Constructor called" + if [ $(echo $container_logs | grep "$indicator" | wc -l) != "0" ]; then + echo "PPM container is ready" + break + fi + sleep 1 - echo "[log] waiting for PPM to start..." done container_logs=$(docker logs $PPM_CONTAINER_NAME 2>&1) if [ $(echo $container_logs | grep "Failed to make shape" | wc -l) != "0" ]; then - echo "[log] Warning: PPM failed to make shape." + echo "Warning: PPM failed to make shape." fi } stopPPMContainer() { if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) != "0" ]; then - echo "[log] stopping existing PPM container" + echo "Stopping existing PPM container" docker stop $PPM_CONTAINER_NAME > /dev/null fi docker rm -f $PPM_CONTAINER_NAME > /dev/null @@ -105,16 +109,14 @@ else fi echo "**************************" -echo "Running standalone test with "$1 $2 $3 $4 +echo "Running standalone test in $PPM_CONTAINER_NAME container with "$1 $2 $3 $4 echo "**************************" startPPMContainer if [ $4 = "BSM" ]; then - echo "[log] performing bsm test in $PPM_CONTAINER_NAME container" docker exec $PPM_CONTAINER_NAME /cvdi-stream/docker-test/do_bsm_test.sh $OFFSET elif [ $4 = "TIM" ]; then - echo "[log] performing tim test in $PPM_CONTAINER_NAME container" docker exec $PPM_CONTAINER_NAME /cvdi-stream/docker-test/do_tim_test.sh $OFFSET fi From 58ae4c8cd3946e082390d8f71544d6dacaea5c0b Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Tue, 9 May 2023 13:59:05 -0600 Subject: [PATCH 08/51] Gave the PPM container an arbitrary 10 seconds to spin up in `standalone.sh` script --- test-scripts/standalone.sh | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/test-scripts/standalone.sh b/test-scripts/standalone.sh index d7b784f9..3cc6fb34 100755 --- a/test-scripts/standalone.sh +++ b/test-scripts/standalone.sh @@ -27,22 +27,13 @@ startPPMContainer() { echo "Starting PPM in new container" docker run --name $PPM_CONTAINER_NAME --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true -v /tmp/docker-test/data:/ppm_data -d -p '8080:8080' $PPM_IMAGE_NAME:$PPM_IMAGE_TAG /cvdi-stream/docker-test/ppm_standalone.sh - # wait until container spins up - while true; do - if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) == "0" ]; then - echo "PPM container '$PPM_CONTAINER_NAME' is not running. Exiting." - exit 1 - fi - - container_logs=$(docker logs $PPM_CONTAINER_NAME 2>&1) - indicator="BSMHandler::BSMHandler(): Constructor called" - if [ $(echo $container_logs | grep "$indicator" | wc -l) != "0" ]; then - echo "PPM container is ready" - break - fi + echo "Giving PPM container time to spin up" + sleep 10 - sleep 1 - done + if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) == "0" ]; then + echo "PPM container '$PPM_CONTAINER_NAME' is not running. Exiting." + exit 1 + fi container_logs=$(docker logs $PPM_CONTAINER_NAME 2>&1) if [ $(echo $container_logs | grep "Failed to make shape" | wc -l) != "0" ]; then From cb3b5a976f0853f6989da91833f646563ac6a3b2 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Tue, 9 May 2023 15:29:28 -0600 Subject: [PATCH 09/51] Refactored `standalone_multi.sh` script and gave PPM containers more time to be ready. --- do_kafka_test.sh | 8 +--- test-scripts/standalone.sh | 8 +++- test-scripts/standalone_multi.sh | 67 ++++++++++++++++++++++++++++---- 3 files changed, 66 insertions(+), 17 deletions(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 1bf16440..a0ade017 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -18,12 +18,6 @@ setup() { exit 1 fi - if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) != "0" ]; then - echo "Stopping existing PPM container" - docker stop $PPM_CONTAINER_NAME > /dev/null - fi - docker rm -f $PPM_CONTAINER_NAME > /dev/null - ./start_kafka.sh } @@ -113,7 +107,7 @@ run_tests() { echo "" echo "" - echo -e $YELLOW"Test 10/$numberOfTests"$NC + echo -e $YELLOW"Test 10/$numberOfTests (2 tests in one)"$NC ./test-scripts/standalone_multi.sh $MAP_FILE config/bsm-test/c6.properties config/tim-test/c3.properties $BSM_DATA_FILE $TIM_DATA_FILE 48 23 } diff --git a/test-scripts/standalone.sh b/test-scripts/standalone.sh index 3cc6fb34..fe836840 100755 --- a/test-scripts/standalone.sh +++ b/test-scripts/standalone.sh @@ -10,8 +10,11 @@ PPM_CONTAINER_NAME=ppm_kafka PPM_IMAGE_TAG=do-kafka-test-ppm-image PPM_IMAGE_NAME=jpo-cvdp_ppm +SECONDS_TO_WAIT_FOR_PPM_READINESS=20 startPPMContainer() { + stopPPMContainer + # Start the PPM in a new container. dockerHostIp=$DOCKER_HOST_IP @@ -27,8 +30,8 @@ startPPMContainer() { echo "Starting PPM in new container" docker run --name $PPM_CONTAINER_NAME --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true -v /tmp/docker-test/data:/ppm_data -d -p '8080:8080' $PPM_IMAGE_NAME:$PPM_IMAGE_TAG /cvdi-stream/docker-test/ppm_standalone.sh - echo "Giving PPM container time to spin up" - sleep 10 + echo "Giving $PPM_CONTAINER_NAME $SECONDS_TO_WAIT_FOR_PPM_READINESS seconds to spin up" + sleep $SECONDS_TO_WAIT_FOR_PPM_READINESS if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) == "0" ]; then echo "PPM container '$PPM_CONTAINER_NAME' is not running. Exiting." @@ -105,6 +108,7 @@ echo "**************************" startPPMContainer +# Produce the test data. if [ $4 = "BSM" ]; then docker exec $PPM_CONTAINER_NAME /cvdi-stream/docker-test/do_bsm_test.sh $OFFSET elif [ $4 = "TIM" ]; then diff --git a/test-scripts/standalone_multi.sh b/test-scripts/standalone_multi.sh index 71c108fc..1b98f022 100755 --- a/test-scripts/standalone_multi.sh +++ b/test-scripts/standalone_multi.sh @@ -4,8 +4,59 @@ # only use this for testing/demo; NOT production. # Test two ppm modules at once, as two docker containers. + +PPM_BSM_CONTAINER_NAME=ppm_bsm_kafka +PPM_TIM_CONTAINER_NAME=ppm_tim_kafka +PPM_IMAGE_TAG=do-kafka-test-ppm-image +PPM_IMAGE_NAME=jpo-cvdp_ppm +SECONDS_TO_WAIT_FOR_PPM_READINESS=20 + USAGE="standalone_multi.sh [MAP_FILE] [BSM_CONFIG] [TIM_CONFIG] [BSM_TEST_FILE] [TIM_TEST_FILE] [BSM_OFFSET] [TIM_OFSET]" +startPPMContainer() { + # Start the PPM in a new container. + dockerHostIp=$DOCKER_HOST_IP + PPM_CONTAINER_NAME=$1 + data_source=$2 + ppm_container_port=$3 + + stopPPMContainer $PPM_CONTAINER_NAME + + # make sure ip can be pinged + while true; do + if ping -c 1 $dockerHostIp &> /dev/null; then + break + else + echo "Docker host ip $dockerHostIp is not pingable. Exiting." + exit 1 + fi + done + echo "Starting PPM in new container" + docker run --name $PPM_CONTAINER_NAME --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true -v $data_source:/ppm_data -d -p $ppm_container_port':8080' $PPM_IMAGE_NAME:$PPM_IMAGE_TAG /cvdi-stream/docker-test/ppm_standalone.sh + + echo "Giving $PPM_CONTAINER_NAME $SECONDS_TO_WAIT_FOR_PPM_READINESS seconds to spin up" + sleep $SECONDS_TO_WAIT_FOR_PPM_READINESS + + if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) == "0" ]; then + echo "PPM container '$PPM_CONTAINER_NAME' is not running. Exiting." + exit 1 + fi + + container_logs=$(docker logs $PPM_CONTAINER_NAME 2>&1) + if [ $(echo $container_logs | grep "Failed to make shape" | wc -l) != "0" ]; then + echo "Warning: PPM failed to make shape." + fi +} + +stopPPMContainer() { + PPM_CONTAINER_NAME=$1 + if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) != "0" ]; then + echo "Stopping existing PPM container" + docker stop $PPM_CONTAINER_NAME > /dev/null + fi + docker rm -f $PPM_CONTAINER_NAME > /dev/null +} + if [ -z $1 ] || [ ! -f $1 ]; then echo "Map file: "$1" not found!" echo $USAGE @@ -68,16 +119,16 @@ echo "**************************" echo "Running standalone multi PPM test with "$1 $2 $3 $4 $5 $6 $7 echo "**************************" +dockerHostIp=$DOCKER_HOST_IP # Start the BSM PPM in a new container. -docker run --name ppm_bsm_kafka -v /tmp/docker-test/bsm-data:/ppm_data -it --rm -p '8080:8080' -d jpo-cvdp_ppm:latest /cvdi-stream/docker-test/ppm_standalone.sh > /dev/null -# Start the TIM PPM in a new container. -docker run --name ppm_tim_kafka -v /tmp/docker-test/tim-data:/ppm_data -it --rm -p '8081:8080' -d jpo-cvdp_ppm:latest /cvdi-stream/docker-test/ppm_standalone.sh > /dev/null +startPPMContainer $PPM_BSM_CONTAINER_NAME /tmp/docker-test/bsm-data 8080 -sleep 10 +# Start the TIM PPM in a new container. +startPPMContainer $PPM_TIM_CONTAINER_NAME /tmp/docker-test/tim-data 8081 # Produce the test data. -docker exec ppm_bsm_kafka /cvdi-stream/docker-test/do_bsm_test.sh $BSM_OFFSET -docker exec ppm_tim_kafka /cvdi-stream/docker-test/do_tim_test.sh $TIM_OFFSET +docker exec $PPM_BSM_CONTAINER_NAME /cvdi-stream/docker-test/do_bsm_test.sh $BSM_OFFSET +docker exec $PPM_TIM_CONTAINER_NAME /cvdi-stream/docker-test/do_tim_test.sh $TIM_OFFSET -docker stop ppm_bsm_kafka > /dev/null -docker stop ppm_tim_kafka > /dev/null +stopPPMContainer $PPM_BSM_CONTAINER_NAME +stopPPMContainer $PPM_TIM_CONTAINER_NAME From 2685f496d251733a536f62b5cd709964f2ec29e5 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Tue, 9 May 2023 16:45:31 -0600 Subject: [PATCH 10/51] Added descriptions as comments to a number of scripts. --- do_kafka_test.sh | 8 ++++++++ docker-test/do_bsm_test.sh | 7 +++++++ docker-test/do_tim_test.sh | 7 +++++++ docker-test/ppm_standalone.sh | 4 ++++ test-scripts/standalone.sh | 21 ++++++++++++++------- test-scripts/standalone_multi.sh | 9 ++++++--- 6 files changed, 46 insertions(+), 10 deletions(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index a0ade017..f6f0e7d8 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -1,5 +1,13 @@ #!/bin/bash +# This script tests the PPM against a kafka cluster. It sets up variables for container and input data +# file names. It starts a Kafka container using another script and checks that required topics are created. +# If the container or topics are missing, the script exits. It builds a Docker image using the current +# directory and specified name/tag. It runs a series of tests using a script with different properties +# and input data files, outputting results to the console. It stops the Kafka container after the tests +# are completed. The script performs five steps: set up the test environment, wait for Kafka to create +# topics, build the PPM image, run the tests, and clean up. + CYAN='\033[0;36m' YELLOW='\033[1;33m' NC='\033[0m' # No Color diff --git a/docker-test/do_bsm_test.sh b/docker-test/do_bsm_test.sh index fb399bac..2c7c67a9 100755 --- a/docker-test/do_bsm_test.sh +++ b/docker-test/do_bsm_test.sh @@ -1,4 +1,11 @@ #!/bin/bash + +# This script produces and consumes messages from Kafka topics. It reads a JSON file containing raw +# BSM data, processes it with a Python script, and then sends the output to a Kafka topic. Then it +# consumes the filtered messages from the Kafka topic, using a specified offset, and checks if +# any messages were received. If no messages were received after a certain number of attempts, the +# script exits with an error message. Otherwise, the script exits with a success message. + export LD_LIBRARY_PATH=/usr/local/lib RED='\033[0;31m' diff --git a/docker-test/do_tim_test.sh b/docker-test/do_tim_test.sh index ca44d2d7..a133e877 100755 --- a/docker-test/do_tim_test.sh +++ b/docker-test/do_tim_test.sh @@ -1,4 +1,11 @@ #!/bin/bash + +# This script produces and consumes messages from Kafka topics. It reads a JSON file containing raw +# TIM data, processes it with a Python script, and then sends the output to a Kafka topic. Then it +# consumes the filtered messages from the Kafka topic, using a specified offset, and checks if +# any messages were received. If no messages were received after a certain number of attempts, the +# script exits with an error message. Otherwise, the script exits with a success message. + export LD_LIBRARY_PATH=/usr/local/lib RED='\033[0;31m' diff --git a/docker-test/ppm_standalone.sh b/docker-test/ppm_standalone.sh index 43aba121..ddb935e7 100755 --- a/docker-test/ppm_standalone.sh +++ b/docker-test/ppm_standalone.sh @@ -1,4 +1,8 @@ #!/bin/sh + +# This script starts the PPM tool with the specified configuration file, using the DOCKER_HOST_IP +# environment variable to connect to a Kafka broker. + export LD_LIBRARY_PATH=/usr/local/lib if [ -z "$DOCKER_HOST_IP" ]; then diff --git a/test-scripts/standalone.sh b/test-scripts/standalone.sh index fe836840..3e4c5a66 100755 --- a/test-scripts/standalone.sh +++ b/test-scripts/standalone.sh @@ -1,11 +1,18 @@ #!/bin/bash -# NOTE that this script will hang if the offsets are wrong. In other words, -# only use this for testing/demo; NOT production. +# This script sets up and runs a standalone test for a PPM container. The +# PPM is started in a Docker container using a specified image and waits for it to become +# ready. The script takes in three input files: ROAD_FILE, CONFIG, TEST_DATA, and a type +# argument (BSM or TIM). It checks if the input files exist and copies them to a test data +# directory. If the OFFSET argument is provided, it is used as the offset in the topic that +# will be consumed and displayed in the output. If not, the default value of 0 is used. -# There are three input files: ROAD_FILE, CONFIG, TEST_DATA. -# Offset is the offset in the topic that will be consumed and displayed in the -# output +# The script then produces the test data by executing either do_bsm_test.sh or do_tim_test.sh +# depending on the type argument, passing in the OFFSET value as an argument. The PPM container +# is then stopped, and the script ends. + +# This script should only be used for testing or demo purposes, as it may hang if the offsets are wrong. It also +# checks if the required configuration and test data files exist before proceeding with the test. PPM_CONTAINER_NAME=ppm_kafka PPM_IMAGE_TAG=do-kafka-test-ppm-image @@ -27,7 +34,7 @@ startPPMContainer() { exit 1 fi done - echo "Starting PPM in new container" + echo "Starting PPM in new container '$PPM_CONTAINER_NAME'" docker run --name $PPM_CONTAINER_NAME --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true -v /tmp/docker-test/data:/ppm_data -d -p '8080:8080' $PPM_IMAGE_NAME:$PPM_IMAGE_TAG /cvdi-stream/docker-test/ppm_standalone.sh echo "Giving $PPM_CONTAINER_NAME $SECONDS_TO_WAIT_FOR_PPM_READINESS seconds to spin up" @@ -46,7 +53,7 @@ startPPMContainer() { stopPPMContainer() { if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) != "0" ]; then - echo "Stopping existing PPM container" + echo "Stopping existing PPM container '$PPM_CONTAINER_NAME'" docker stop $PPM_CONTAINER_NAME > /dev/null fi docker rm -f $PPM_CONTAINER_NAME > /dev/null diff --git a/test-scripts/standalone_multi.sh b/test-scripts/standalone_multi.sh index 1b98f022..ae937498 100755 --- a/test-scripts/standalone_multi.sh +++ b/test-scripts/standalone_multi.sh @@ -1,9 +1,12 @@ #!/bin/bash -# NOTE that this script will hang if the offsets are wrong. In other words, -# only use this for testing/demo; NOT production. +# This script starts two PPM containers, one for BSMs and one for TIMs, in separate Docker containers. +# It first checks if all necessary input files exist, creates necessary directories, and copies the input +# files to those directories. It then starts the PPM containers, waits for them to spin up, produces test +# data, and finally stops the containers. -# Test two ppm modules at once, as two docker containers. +# This script should only be used for testing or demo purposes, as it may hang if the offsets are wrong. It also +# checks if the required configuration and test data files exist before proceeding with the test. PPM_BSM_CONTAINER_NAME=ppm_bsm_kafka PPM_TIM_CONTAINER_NAME=ppm_tim_kafka From adfb334000538d801ca4fc5b2873b177f13bde8e Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Tue, 9 May 2023 16:54:04 -0600 Subject: [PATCH 11/51] Removed duplicate python installation in `Dockerfile.standalone` --- Dockerfile.standalone | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile.standalone b/Dockerfile.standalone index 36f1372e..a01fe6f4 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -27,8 +27,5 @@ ADD ./unit-test-data /cvdi-stream/unit-test-data # Do the build. RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make -# install python for testing -RUN apt-get install -y python3 - # Add test data. This changes frequently so keep it low in the file. ADD ./docker-test /cvdi-stream/docker-test From b8b1124b6f61d9c0bdab6c69e1f1e792475eb6e9 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Fri, 16 Jun 2023 13:53:27 -0600 Subject: [PATCH 12/51] Created separate dockerfile for testing. --- Dockerfile | 3 --- Dockerfile.testing | 35 +++++++++++++++++++++++++++++++++++ do_kafka_test.sh | 2 +- 3 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 Dockerfile.testing diff --git a/Dockerfile b/Dockerfile index 284fe674..600551cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,9 +13,6 @@ RUN apt install -y cmake # Install librdkafka. RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev -# install python for testing -RUN apt-get install -y python3 - # add the source and build files ADD CMakeLists.txt /cvdi-stream ADD ./src /cvdi-stream/src diff --git a/Dockerfile.testing b/Dockerfile.testing new file mode 100644 index 00000000..284fe674 --- /dev/null +++ b/Dockerfile.testing @@ -0,0 +1,35 @@ +FROM ubuntu:18.04 +USER root + +WORKDIR /cvdi-stream + +# Add build tools. +RUN apt-get update && apt-get install -y g++ + +# Install cmake. +RUN apt install -y libprotobuf-dev protobuf-compiler +RUN apt install -y cmake + +# Install librdkafka. +RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev + +# install python for testing +RUN apt-get install -y python3 + +# add the source and build files +ADD CMakeLists.txt /cvdi-stream +ADD ./src /cvdi-stream/src +ADD ./cv-lib /cvdi-stream/cv-lib +ADD ./include /cvdi-stream/include +ADD ./kafka-test /cvdi-stream/kafka-test +ADD ./unit-test-data /cvdi-stream/unit-test-data + +# Do the build. +RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make + +# Add test data. This changes frequently so keep it low in the file. +ADD ./docker-test /cvdi-stream/docker-test + +# Run the tool. +RUN chmod 7777 /cvdi-stream/docker-test/ppm_no_map.sh +CMD ["/cvdi-stream/docker-test/ppm_no_map.sh"] \ No newline at end of file diff --git a/do_kafka_test.sh b/do_kafka_test.sh index f6f0e7d8..1c19dfe8 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -59,7 +59,7 @@ waitForKafkaToCreateTopics() { } buildPPMImage() { - docker build . -t $PPM_IMAGE_NAME:$PPM_IMAGE_TAG + docker build . -t $PPM_IMAGE_NAME:$PPM_IMAGE_TAG -f Dockerfile.testing } run_tests() { From 4dbfdb087bb6fb8355ed05972dbefa9b0851cd40 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Mon, 19 Jun 2023 11:41:42 -0600 Subject: [PATCH 13/51] Removed python installation from `Dockerfile-nsv` and `Dockerfile.standalone` --- Dockerfile-nsv | 3 --- Dockerfile.standalone | 3 --- 2 files changed, 6 deletions(-) diff --git a/Dockerfile-nsv b/Dockerfile-nsv index 9d178d02..69cd0977 100644 --- a/Dockerfile-nsv +++ b/Dockerfile-nsv @@ -15,9 +15,6 @@ RUN apt install -y cmake # Install librdkafka. RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev -# install python for testing -RUN apt-get install -y python3 - # add the source and build files ADD CMakeLists.txt /cvdi-stream ADD ./src /cvdi-stream/src diff --git a/Dockerfile.standalone b/Dockerfile.standalone index a01fe6f4..0dcefde7 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -13,9 +13,6 @@ RUN apt install -y cmake # Install librdkafka. RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev -# install python for testing -RUN apt-get install -y python3 - # add the source and build files ADD CMakeLists.txt /cvdi-stream ADD ./src /cvdi-stream/src From 96976ffb6811aaaa7b099c2c65be5a1e70b44144 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Mon, 19 Jun 2023 14:46:57 -0600 Subject: [PATCH 14/51] Added documentation for the `do_kafka_test.sh` script to the README. --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 12e59fd6..e05c8157 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,17 @@ When running the project in the provided dev container, the REDACTION_PROPERTIES #### RPM Debug If the RPM_DEBUG environment variable is set to true, debug messages will be logged to a file by the RedactionPropertiesManager class. This will allow developers to see whether the environment variable is set, whether the file was found and whether a non-zero number of redaction fields were loaded in. +## Kafka Test Script +The [do_kafka_test.sh](./do_kafka_test.sh) script is designed to perform integration tests on a Kafka instance. To execute the tests, this script relies on the following scripts: standalone.sh, standalone_multi.sh, do_bsm_test.sh, and do_tim_test.sh. + +To ensure proper execution, it is recommended to run this script in WSL (Windows Subsystem for Linux). + +The DOCKER_HOST_IP environment variable must be set to the IP address of the host machine. This is required for the script to function properly. This can be set by using the following command: + +``` +export DOCKER_HOST_IP=$(ifconfig | zgrep -m 1 -oP '(?<=inet\s)\d+(\.\d+){3}') +``` + ## Some Notes - The tests for this project can be run after compilation by running the "ppm_tests" executable. - When manually compiling with WSL, librdkafka will sometimes not be recognized. This can be resolved by utilizing the provided dev environment. From 42cda70cfc583d9da5b94a8462f71939e5718f14 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Fri, 23 Jun 2023 09:12:56 -0600 Subject: [PATCH 15/51] Modified 'Kafka Test Script' section of README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f34a31d2..9460ea6c 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,7 @@ The [`build_and_exec.sh`](./build_and_exec.sh) script can be used to build a tag ## Kafka Test Script The [do_kafka_test.sh](./do_kafka_test.sh) script is designed to perform integration tests on a Kafka instance. To execute the tests, this script relies on the following scripts: standalone.sh, standalone_multi.sh, do_bsm_test.sh, and do_tim_test.sh. -To ensure proper execution, it is recommended to run this script in WSL (Windows Subsystem for Linux). +To ensure proper execution, it is recommended to run this script outside of the dev container where docker is available. This is because the script will spin up a standalone kafka instance and will not be able to access the docker daemon from within the dev container. The DOCKER_HOST_IP environment variable must be set to the IP address of the host machine. This is required for the script to function properly. This can be set by using the following command: From a22253b834801207abc9a1f64555f7e9ed213803 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Fri, 23 Jun 2023 09:16:50 -0600 Subject: [PATCH 16/51] Modified `Build & Exec Script` section of the README. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9460ea6c..e0565f3a 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,8 @@ If the RPM_DEBUG environment variable is set to true, debug messages will be log ## Build & Exec Script The [`build_and_exec.sh`](./build_and_exec.sh) script can be used to build a tagged image of the PPM, run the container & enter it with an interactive shell. This script can be used to test the PPM in a standalone environment. +This script should be run outside of the dev container in an environment where Docker is available. + ## Kafka Test Script The [do_kafka_test.sh](./do_kafka_test.sh) script is designed to perform integration tests on a Kafka instance. To execute the tests, this script relies on the following scripts: standalone.sh, standalone_multi.sh, do_bsm_test.sh, and do_tim_test.sh. From f1a4e4c5bf0c00717166dc59ae41f027f154f5e8 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Fri, 23 Jun 2023 09:50:45 -0600 Subject: [PATCH 17/51] Added note on using the LF EOF sequence for scripts to the README. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index e0565f3a..7c75ac5c 100644 --- a/README.md +++ b/README.md @@ -212,11 +212,15 @@ The [`build_and_exec.sh`](./build_and_exec.sh) script can be used to build a tag This script should be run outside of the dev container in an environment where Docker is available. +It should be noted that this script needs to use the LF end-of-line sequence. + ## Kafka Test Script The [do_kafka_test.sh](./do_kafka_test.sh) script is designed to perform integration tests on a Kafka instance. To execute the tests, this script relies on the following scripts: standalone.sh, standalone_multi.sh, do_bsm_test.sh, and do_tim_test.sh. To ensure proper execution, it is recommended to run this script outside of the dev container where docker is available. This is because the script will spin up a standalone kafka instance and will not be able to access the docker daemon from within the dev container. +It should be noted that this file needs to use the LF end-of-line sequence. + The DOCKER_HOST_IP environment variable must be set to the IP address of the host machine. This is required for the script to function properly. This can be set by using the following command: ``` From 1ab0ac773affc92ae48da1c760a2189f1b025a45 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Fri, 23 Jun 2023 10:14:21 -0600 Subject: [PATCH 18/51] Updated base image in dockerfiles to ubuntu:20.04 --- Dockerfile | 2 +- Dockerfile-nsv | 2 +- Dockerfile.standalone | 2 +- Dockerfile.testing | 2 +- README.md | 9 ++++++++- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index d2a13c22..1b4958ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 USER root WORKDIR /cvdi-stream diff --git a/Dockerfile-nsv b/Dockerfile-nsv index 7b8a9338..62d7c0c5 100644 --- a/Dockerfile-nsv +++ b/Dockerfile-nsv @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 USER root ARG PPM_CONFIG_FILE ARG PPM_MAP_FILE diff --git a/Dockerfile.standalone b/Dockerfile.standalone index 43753293..69ee498d 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 USER root WORKDIR /cvdi-stream diff --git a/Dockerfile.testing b/Dockerfile.testing index 284fe674..b2e8ceeb 100644 --- a/Dockerfile.testing +++ b/Dockerfile.testing @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 USER root WORKDIR /cvdi-stream diff --git a/README.md b/README.md index 7c75ac5c..89f23b4e 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,14 @@ The [do_kafka_test.sh](./do_kafka_test.sh) script is designed to perform integra To ensure proper execution, it is recommended to run this script outside of the dev container where docker is available. This is because the script will spin up a standalone kafka instance and will not be able to access the docker daemon from within the dev container. -It should be noted that this file needs to use the LF end-of-line sequence. +It should be noted that this script and any dependent scripts need to use the LF end-of-line sequence. These include the following: +- do_kafka_test.sh +- standalone.sh +- standalone_multi.sh +- do_bsm_test.sh +- do_tim_test.sh +- test_in.py +- test_out.py The DOCKER_HOST_IP environment variable must be set to the IP address of the host machine. This is required for the script to function properly. This can be set by using the following command: From 7b362ac3e8744e24a5f79787b8fa3f2544611928 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Fri, 23 Jun 2023 10:58:35 -0600 Subject: [PATCH 19/51] Copied 'config' directory over to the image during build. --- Dockerfile | 3 +++ Dockerfile-nsv | 3 +++ Dockerfile.standalone | 3 +++ Dockerfile.testing | 5 ++++- 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1b4958ef..f79c115d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,8 @@ USER root WORKDIR /cvdi-stream +ENV DEBIAN_FRONTEND=noninteractive + # Add build tools. RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100 @@ -26,6 +28,7 @@ ADD ./cv-lib /cvdi-stream/cv-lib ADD ./include /cvdi-stream/include ADD ./kafka-test /cvdi-stream/kafka-test ADD ./unit-test-data /cvdi-stream/unit-test-data +ADD ./config /cvdi-stream/config # Do the build. RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make diff --git a/Dockerfile-nsv b/Dockerfile-nsv index 62d7c0c5..a83a6334 100644 --- a/Dockerfile-nsv +++ b/Dockerfile-nsv @@ -5,6 +5,8 @@ ARG PPM_MAP_FILE WORKDIR /cvdi-stream +ENV DEBIAN_FRONTEND=noninteractive + # Add build tools. RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100 @@ -30,6 +32,7 @@ ADD ./kafka-test /cvdi-stream/kafka-test ADD ./unit-test-data /cvdi-stream/unit-test-data ADD config/${PPM_CONFIG_FILE} /cvdi-stream/config/ ADD data/${PPM_MAP_FILE} /cvdi-stream/config/ +ADD ./config /cvdi-stream/config # Do the build. RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make diff --git a/Dockerfile.standalone b/Dockerfile.standalone index 69ee498d..9d1e0d4b 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -3,6 +3,8 @@ USER root WORKDIR /cvdi-stream +ENV DEBIAN_FRONTEND=noninteractive + # Add build tools. RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100 @@ -26,6 +28,7 @@ ADD ./cv-lib /cvdi-stream/cv-lib ADD ./include /cvdi-stream/include ADD ./kafka-test /cvdi-stream/kafka-test ADD ./unit-test-data /cvdi-stream/unit-test-data +ADD ./config /cvdi-stream/config # Do the build. RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make diff --git a/Dockerfile.testing b/Dockerfile.testing index b2e8ceeb..63684003 100644 --- a/Dockerfile.testing +++ b/Dockerfile.testing @@ -3,8 +3,10 @@ USER root WORKDIR /cvdi-stream +ENV DEBIAN_FRONTEND=noninteractive + # Add build tools. -RUN apt-get update && apt-get install -y g++ +RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100 # Install cmake. RUN apt install -y libprotobuf-dev protobuf-compiler @@ -23,6 +25,7 @@ ADD ./cv-lib /cvdi-stream/cv-lib ADD ./include /cvdi-stream/include ADD ./kafka-test /cvdi-stream/kafka-test ADD ./unit-test-data /cvdi-stream/unit-test-data +ADD ./config /cvdi-stream/config # Do the build. RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make From 30ca2af7538eb4554d04b4ca038aab1e540f7bbd Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Fri, 23 Jun 2023 11:55:39 -0600 Subject: [PATCH 20/51] Switched back to using ubuntu:18.04 for the base image. --- Dockerfile | 2 +- Dockerfile-nsv | 2 +- Dockerfile.standalone | 2 +- Dockerfile.testing | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f79c115d..a6e181ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:18.04 USER root WORKDIR /cvdi-stream diff --git a/Dockerfile-nsv b/Dockerfile-nsv index a83a6334..c9b796e0 100644 --- a/Dockerfile-nsv +++ b/Dockerfile-nsv @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:18.04 USER root ARG PPM_CONFIG_FILE ARG PPM_MAP_FILE diff --git a/Dockerfile.standalone b/Dockerfile.standalone index 9d1e0d4b..3c29b2d0 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:18.04 USER root WORKDIR /cvdi-stream diff --git a/Dockerfile.testing b/Dockerfile.testing index 63684003..47c14f3b 100644 --- a/Dockerfile.testing +++ b/Dockerfile.testing @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:18.04 USER root WORKDIR /cvdi-stream From 0300a34e774ddbe3b2e9966d2867ad5dfafdfef1 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Fri, 23 Jun 2023 13:15:00 -0600 Subject: [PATCH 21/51] Switched over to using $DOCKER_HOST_IP instead of `172.17.0.1` as input to the kafka-topics.sh script. --- do_kafka_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 1c19dfe8..44ae7c43 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -37,7 +37,7 @@ waitForKafkaToCreateTopics() { exit 1 fi - ltopics=$(docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1) + ltopics=$(docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper $DOCKER_HOST_IP) allTopicsCreated=true if [ $(echo $ltopics | grep "topic.FilteredOdeBsmJson" | wc -l) == "0" ]; then allTopicsCreated=false From 84e5153b2b4872a3a497d695ec1712341d47f621 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Fri, 23 Jun 2023 15:09:31 -0600 Subject: [PATCH 22/51] Added a note regarding WSL hanging to the README. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 89f23b4e..410491e5 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,8 @@ The DOCKER_HOST_IP environment variable must be set to the IP address of the hos export DOCKER_HOST_IP=$(ifconfig | zgrep -m 1 -oP '(?<=inet\s)\d+(\.\d+){3}') ``` +WSL will sometimes hang indefinitely while script waits for kafka to create topics. When this happens, running `wsl --shutdown` in a windows command prompt and restarting the docker services is recommended. + ## Some Notes - The tests for this project can be run after compilation by running the "ppm_tests" executable. - When manually compiling with WSL, librdkafka will sometimes not be recognized. This can be resolved by utilizing the provided dev environment. From af674cf899af4fc2e8b99dab22281b7c5688e25e Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Fri, 23 Jun 2023 16:49:43 -0600 Subject: [PATCH 23/51] Switched back to using 172.17.0.1 instead of $DOCKER_HOST_IP for the topic creation check. --- do_kafka_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 44ae7c43..1c19dfe8 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -37,7 +37,7 @@ waitForKafkaToCreateTopics() { exit 1 fi - ltopics=$(docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper $DOCKER_HOST_IP) + ltopics=$(docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1) allTopicsCreated=true if [ $(echo $ltopics | grep "topic.FilteredOdeBsmJson" | wc -l) == "0" ]; then allTopicsCreated=false From 68a9c80d257f1f704e0ec350a8649406c754c474 Mon Sep 17 00:00:00 2001 From: Saikrishna Bairamoni <84093461+SaikrishnaBairamoni@users.noreply.github.com> Date: Fri, 7 Jul 2023 09:11:21 -0400 Subject: [PATCH 24/51] initial gha ci workflow and sonar scan --- .github/workflows/ci.yml | 65 ++++++++++++++++++++++++++++++++++++++++ build.sh | 33 ++++++++++++++++++++ sonar-project.properties | 18 +++++++---- 3 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 build.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..5a2efa25 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,65 @@ +name: cvdp +on: + push: + +jobs: + build: + runs-on: ubuntu-latest + container: + image: ubuntu:jammy-20230126 + env: + REDACTION_PROPERTIES_PATH: "/__w/jpo-cvdp/jpo-cvdp/config/fieldsToRedact.txt" + BUILD_WRAPPER_OUT_DIR: "$GITHUB_WORKSPACE/bw-output" + options: "--user root" + + steps: + - name: checkout code + uses: actions/checkout@v2 + - name: Install Deps + run: | + apt update + apt-get -y install sudo wget curl gnupg lsb-release gcovr unzip + sudo apt-get -y install software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + #curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + sudo curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + sudo apt-get -y update + sudo apt-get -y install docker-ce + - name: set up Cmake + uses: jwlawson/actions-setup-cmake@v1.13 + with: + cmake-version: '3.16' + - name: install g++ + run: | + sudo apt-get -y install build-essential + sudo apt -y install cmake g++ libprotobuf-dev protobuf-compiler + sudo apt -y install apt-transport-https ca-certificates curl software-properties-common + export CXX="g++" + - name: install librdkafka + run: | + git clone --depth 1 https://github.com/confluentinc/librdkafka.git librdkafka + cd librdkafka + cmake -H. -B_cmake_build + cmake --build _cmake_build + cmake --build _cmake_build --target install + - name: Install sonar-scanner and build-wrapper + uses: SonarSource/sonarcloud-github-c-cpp@v1 + - name: Build and Test + run: | + cd $GITHUB_WORKSPACE + chmod 755 -R /__w/jpo-cvdp/jpo-cvdp + export LD_LIBRARY_PATH=/usr/local/lib + build-wrapper-linux-x86-64 --out-dir $GITHUB_WORKSPACE/bw-output ./build.sh + mkdir coverage + cd coverage + gcov $GITHUB_WORKSPACE/cv-lib/src/*.cpp --object-directory /__w/jpo-cvdp/jpo-cvdp/build/cv-lib/CMakeFiles/CVLib.dir/src/ + gcov $GITHUB_WORKSPACE/src/*.cpp --object-directory /__w/jpo-cvdp/jpo-cvdp/build/CMakeFiles/ppm_tests.dir/src/ + - name: Run sonar-scanner + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + run: | + cd $GITHUB_WORKSPACE + sonar-scanner \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100644 index 00000000..1fdf1f6a --- /dev/null +++ b/build.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright (C) 2018-2020 LEIDOS. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# script executes all kafka_clients and scheduling service build and coverage steps so that they can be singularly +# wrapped by the sonarcloud build-wrapper +set -e + + +# check if REDACTION_PROPERTIES_PATH is set +if [ -z "$REDACTION_PROPERTIES_PATH" ] ; then + echo "REDACTION_PROPERTIES_PATH is not set" + exit 1 +fi + +mkdir build +cd build +cmake .. +cmake --build . + +./ppm_tests +ctest --output-on-failure \ No newline at end of file diff --git a/sonar-project.properties b/sonar-project.properties index 45519b65..3c88524a 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,10 +1,16 @@ -sonar.projectKey=jpo-cvdp-key +sonar.host.url=https://sonarcloud.io sonar.projectName=jpo-cvdp sonar.projectVersion=1.0 -sonar.sources=src,cv-lib/src +sonar.projectKey=usdot-jpo-ode-1_jpo-cvdp +sonar.organization=usdot-jpo-ode-1 +sonar.sources=. sonar.exclusions=src/ppm.cpp sonar.cfamily.build-wrapper-output=bw-output -sonar.host.url=https://sonarqube.com -sonar.organization=aferber-github -sonar.login=feb286a32ec2f510525f9e3e44da3f81b4de57c0 -sonar.cfamily.gcov.reportsPath=./coverage +sonar.cfamily.gcov.reportsPath=/__w/jpo-cvdp/jpo-cvdp/coverage/ +sonar.sonar.projectBaseDir=/home/runner/work/jpo-cvdp/jpo-cvdp/ +sonar.exclusions=**/*.java +sonar.coverage.exclusions=**/*.java +sonar.scm.disabled=true +sonar.scm.provider=git +sonar.sourceEncoding=UTF-8 + From 7c12db60d20c512494110f3bfa38afe3776f40f7 Mon Sep 17 00:00:00 2001 From: Saikrishna Bairamoni <84093461+SaikrishnaBairamoni@users.noreply.github.com> Date: Fri, 7 Jul 2023 11:11:32 -0400 Subject: [PATCH 25/51] restructure and add sonar token --- .github/workflows/ci.yml | 157 ++++++++++++++++++++++++--------------- .travis.yml | 61 --------------- 2 files changed, 96 insertions(+), 122 deletions(-) delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a2efa25..9b5ecb07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,65 +1,100 @@ -name: cvdp +name: jpo-cvdp on: push: jobs: - build: - runs-on: ubuntu-latest - container: - image: ubuntu:jammy-20230126 - env: - REDACTION_PROPERTIES_PATH: "/__w/jpo-cvdp/jpo-cvdp/config/fieldsToRedact.txt" - BUILD_WRAPPER_OUT_DIR: "$GITHUB_WORKSPACE/bw-output" - options: "--user root" - - steps: - - name: checkout code - uses: actions/checkout@v2 - - name: Install Deps - run: | - apt update - apt-get -y install sudo wget curl gnupg lsb-release gcovr unzip - sudo apt-get -y install software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - #curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - sudo curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose - sudo apt-get -y update - sudo apt-get -y install docker-ce - - name: set up Cmake - uses: jwlawson/actions-setup-cmake@v1.13 - with: - cmake-version: '3.16' - - name: install g++ - run: | - sudo apt-get -y install build-essential - sudo apt -y install cmake g++ libprotobuf-dev protobuf-compiler - sudo apt -y install apt-transport-https ca-certificates curl software-properties-common - export CXX="g++" - - name: install librdkafka - run: | - git clone --depth 1 https://github.com/confluentinc/librdkafka.git librdkafka - cd librdkafka - cmake -H. -B_cmake_build - cmake --build _cmake_build - cmake --build _cmake_build --target install - - name: Install sonar-scanner and build-wrapper - uses: SonarSource/sonarcloud-github-c-cpp@v1 - - name: Build and Test - run: | - cd $GITHUB_WORKSPACE - chmod 755 -R /__w/jpo-cvdp/jpo-cvdp - export LD_LIBRARY_PATH=/usr/local/lib - build-wrapper-linux-x86-64 --out-dir $GITHUB_WORKSPACE/bw-output ./build.sh - mkdir coverage - cd coverage - gcov $GITHUB_WORKSPACE/cv-lib/src/*.cpp --object-directory /__w/jpo-cvdp/jpo-cvdp/build/cv-lib/CMakeFiles/CVLib.dir/src/ - gcov $GITHUB_WORKSPACE/src/*.cpp --object-directory /__w/jpo-cvdp/jpo-cvdp/build/CMakeFiles/ppm_tests.dir/src/ - - name: Run sonar-scanner - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: | - cd $GITHUB_WORKSPACE - sonar-scanner \ No newline at end of file + jpo-cvdp: + runs-on: ubuntu-latest + container: + image: ubuntu:jammy-20230126 + env: + REDACTION_PROPERTIES_PATH: "/__w/jpo-cvdp/jpo-cvdp/config/fieldsToRedact.txt" + BUILD_WRAPPER_OUT_DIR: "$GITHUB_WORKSPACE/bw-output" + options: "--user root" + steps: + - name: checkout code + uses: actions/checkout@v2 + - name: Install Deps + run: | + apt update + apt-get -y install sudo wget curl gnupg lsb-release gcovr unzip + sudo apt-get -y install software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + #curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + sudo curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + sudo apt-get -y update + sudo apt-get -y install docker-ce + - name: set up Cmake + uses: jwlawson/actions-setup-cmake@v1.13 + with: + cmake-version: '3.16' + - name: install g++ + run: | + sudo apt-get -y install build-essential + sudo apt -y install cmake g++ libprotobuf-dev protobuf-compiler + sudo apt -y install apt-transport-https ca-certificates curl software-properties-common + export CXX="g++" + - name: install librdkafka + run: | + git clone --depth 1 https://github.com/confluentinc/librdkafka.git librdkafka + cd librdkafka + cmake -H. -B_cmake_build + cmake --build _cmake_build + cmake --build _cmake_build --target install + - name: Install sonar-scanner and build-wrapper + uses: SonarSource/sonarcloud-github-c-cpp@v1 + - name: Build and Generate test coverage + run: | + cd $GITHUB_WORKSPACE + chmod 755 -R /__w/jpo-cvdp/jpo-cvdp + export LD_LIBRARY_PATH=/usr/local/lib + build-wrapper-linux-x86-64 --out-dir $GITHUB_WORKSPACE/bw-output ./build.sh + mkdir coverage + cd coverage + gcov $GITHUB_WORKSPACE/cv-lib/src/*.cpp --object-directory /__w/jpo-cvdp/jpo-cvdp/build/cv-lib/CMakeFiles/CVLib.dir/src/ + gcov $GITHUB_WORKSPACE/src/*.cpp --object-directory /__w/jpo-cvdp/jpo-cvdp/build/CMakeFiles/ppm_tests.dir/src/ + ls -la && pwd + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + with: + name: jpo-cvdp + path: /__w/jpo-cvdp/jpo-cvdp/coverage/ + if-no-files-found: error + - name: Archive buildwrapper output + uses: actions/upload-artifact@v3 + with: + name: jpo-cvdp + path: /home/runner/work/jpo-cvdp/jpo-cvdp/bw-output + - name: Setup SonarScanner + uses: warchant/setup-sonar-scanner@v4 + with: + version: 4.8.0.2856 + - name: Generate sonar properties file + run: | + cat < /tmp/sonar-scanner.properties + sonar.host.url=https://sonarcloud.io + sonar.projectName=jpo-cvdp + sonar.projectVersion=1.0 + sonar.projectKey=usdot-jpo-ode-1_jpo-cvdp + sonar.organization=usdot-jpo-ode-1 + sonar.sources=. + sonar.exclusions=src/ppm.cpp + sonar.cfamily.build-wrapper-output=bw-output + sonar.cfamily.gcov.reportsPath=/__w/jpo-cvdp/jpo-cvdp/coverage/ + sonar.sonar.projectBaseDir=/home/runner/work/jpo-cvdp/jpo-cvdp/ + sonar.exclusions=**/*.java + sonar.coverage.exclusions=**/*.java + # Set Git as SCM sensor + sonar.scm.disabled=true + #sonar.scm.enabled=false + sonar.scm.provider=git + sonar.sourceEncoding=UTF-8 + EOF + - name: Run SonarScanner + uses: usdot-fhwa-stol/actions/sonar-scanner@main + with: + sonar-properties-path: /tmp/sonar-scanner.properties + sonar-token: ${{ secrets.SONAR_TOKEN }} + working-dir: $GITHUB_WORKSPACE \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0f146914..00000000 --- a/.travis.yml +++ /dev/null @@ -1,61 +0,0 @@ -language: cpp -sudo: required - -compiler: - - g++ - -services: - - docker - -env: - DOCKER_COMPOSE_VERSION: 1.11.2 - -addons: - sonarqube: - organization: "aferber-github" - token: - secure: $SONAR_SECURITY_TOKEN_NEW - branches: - - .* - -before_install: - - docker version - - docker-compose version - - sudo apt-get update - - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - - sudo rm /usr/local/bin/docker-compose - - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - - chmod +x docker-compose - - sudo mv docker-compose /usr/local/bin - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - - sudo apt-get update -qq - -install: - - sudo apt-get install -qq g++-4.9 - - export CXX="g++-4.9" - -script: - - git clone https://github.com/edenhill/librdkafka.git - - cd librdkafka - - ./configure - - make - - sudo make install - - cd .. - - mkdir /tmp/build - - cd /tmp/build - - cmake $TRAVIS_BUILD_DIR - - export LD_LIBRARY_PATH=/usr/local/lib - - build-wrapper-linux-x86-64 --out-dir $TRAVIS_BUILD_DIR/bw-output make clean all - - ./ppm_tests - - cd $TRAVIS_BUILD_DIR - - mkdir coverage - - cd coverage - - gcov $TRAVIS_BUILD_DIR/cv-lib/src/*.cpp --object-directory /tmp/build/cv-lib/CMakeFiles/CVLib.dir/src/ - - gcov $TRAVIS_BUILD_DIR/src/*.cpp --object-directory /tmp/build/CMakeFiles/ppm_tests.dir/src/ - - cd $TRAVIS_BUILD_DIR - - sonar-scanner - #- ./do_kafka_test.sh - -cache: - directories: - - '$HOME/.sonar/cache' From 399b9f150644bfd13bd23b4a124ee7ba1378da31 Mon Sep 17 00:00:00 2001 From: Saikrishna Bairamoni <84093461+SaikrishnaBairamoni@users.noreply.github.com> Date: Fri, 7 Jul 2023 11:19:33 -0400 Subject: [PATCH 26/51] tweak altStackMe value --- include/catch/catch.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/catch/catch.hpp b/include/catch/catch.hpp index 599eee90..a0ecd16d 100644 --- a/include/catch/catch.hpp +++ b/include/catch/catch.hpp @@ -6462,7 +6462,7 @@ namespace Catch { static bool isSet; static struct sigaction oldSigActions [sizeof(signalDefs)/sizeof(SignalDefs)]; static stack_t oldSigStack; - static char altStackMem[SIGSTKSZ]; + static char altStackMem[32768]; static void handleSignal( int sig ) { std::string name = ""; @@ -6482,7 +6482,7 @@ namespace Catch { isSet = true; stack_t sigStack; sigStack.ss_sp = altStackMem; - sigStack.ss_size = SIGSTKSZ; + sigStack.ss_size = 32768; sigStack.ss_flags = 0; sigaltstack(&sigStack, &oldSigStack); struct sigaction sa = { 0 }; @@ -6513,7 +6513,7 @@ namespace Catch { bool FatalConditionHandler::isSet = false; struct sigaction FatalConditionHandler::oldSigActions[sizeof(signalDefs)/sizeof(SignalDefs)] = {}; stack_t FatalConditionHandler::oldSigStack = {}; - char FatalConditionHandler::altStackMem[SIGSTKSZ] = {}; + char FatalConditionHandler::altStackMem[32768] = {}; } // namespace Catch From e2487fda68818d709fb7a07463ae12c4e30e5fbe Mon Sep 17 00:00:00 2001 From: Saikrishna Bairamoni <84093461+SaikrishnaBairamoni@users.noreply.github.com> Date: Mon, 10 Jul 2023 10:33:01 -0400 Subject: [PATCH 27/51] Update ci.yml --- .github/workflows/ci.yml | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b5ecb07..f9fd6977 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,12 +8,12 @@ jobs: container: image: ubuntu:jammy-20230126 env: - REDACTION_PROPERTIES_PATH: "/__w/jpo-cvdp/jpo-cvdp/config/fieldsToRedact.txt" - BUILD_WRAPPER_OUT_DIR: "$GITHUB_WORKSPACE/bw-output" + REDACTION_PROPERTIES_PATH: "/__w/jpo-cvdp/jpo-cvdp/config/fieldsToRedact.txt" # This env vairable is used to run ppm test + BUILD_WRAPPER_OUT_DIR: "$GITHUB_WORKSPACE/bw-output" # This env variable is needed to run SonarSource/sonarcloud-github-c-cpp@v1 options: "--user root" steps: - name: checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v2 # This action to checkout the code - name: Install Deps run: | apt update @@ -21,13 +21,12 @@ jobs: sudo apt-get -y install software-properties-common curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - #curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose sudo curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose sudo apt-get -y update sudo apt-get -y install docker-ce - name: set up Cmake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.13 # this action is used to setup and install Cmake with required versions with: cmake-version: '3.16' - name: install g++ @@ -36,7 +35,7 @@ jobs: sudo apt -y install cmake g++ libprotobuf-dev protobuf-compiler sudo apt -y install apt-transport-https ca-certificates curl software-properties-common export CXX="g++" - - name: install librdkafka + - name: install librdkafka # This is to install librdkafka package run: | git clone --depth 1 https://github.com/confluentinc/librdkafka.git librdkafka cd librdkafka @@ -44,7 +43,7 @@ jobs: cmake --build _cmake_build cmake --build _cmake_build --target install - name: Install sonar-scanner and build-wrapper - uses: SonarSource/sonarcloud-github-c-cpp@v1 + uses: SonarSource/sonarcloud-github-c-cpp@v1 # This Action Installs sonar cloud and build wrapper to run sonar scan analysis - name: Build and Generate test coverage run: | cd $GITHUB_WORKSPACE @@ -54,21 +53,20 @@ jobs: mkdir coverage cd coverage gcov $GITHUB_WORKSPACE/cv-lib/src/*.cpp --object-directory /__w/jpo-cvdp/jpo-cvdp/build/cv-lib/CMakeFiles/CVLib.dir/src/ - gcov $GITHUB_WORKSPACE/src/*.cpp --object-directory /__w/jpo-cvdp/jpo-cvdp/build/CMakeFiles/ppm_tests.dir/src/ - ls -la && pwd + gcov $GITHUB_WORKSPACE/src/*.cpp --object-directory /__w/jpo-cvdp/jpo-cvdp/build/CMakeFiles/ppm_tests.dir/src/ - name: Archive code coverage results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v3 # This action is used to capture the test artifacts and exits if no files are found with: name: jpo-cvdp path: /__w/jpo-cvdp/jpo-cvdp/coverage/ if-no-files-found: error - name: Archive buildwrapper output - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v3 # This action is used to capture the builwrapper output files used by sonarscan. with: name: jpo-cvdp path: /home/runner/work/jpo-cvdp/jpo-cvdp/bw-output - name: Setup SonarScanner - uses: warchant/setup-sonar-scanner@v4 + uses: warchant/setup-sonar-scanner@v4 # This action is used to setup sonar scanner with required versions. with: version: 4.8.0.2856 - name: Generate sonar properties file @@ -93,8 +91,8 @@ jobs: sonar.sourceEncoding=UTF-8 EOF - name: Run SonarScanner - uses: usdot-fhwa-stol/actions/sonar-scanner@main + uses: usdot-fhwa-stol/actions/sonar-scanner@main # This action runs the analysis using generated sonar gcov files and publish reports to respective sonarcloud project. with: sonar-properties-path: /tmp/sonar-scanner.properties sonar-token: ${{ secrets.SONAR_TOKEN }} - working-dir: $GITHUB_WORKSPACE \ No newline at end of file + working-dir: $GITHUB_WORKSPACE From 76de63654f7f0d1dbb1b18a48e0fe3985be6a954 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Tue, 11 Jul 2023 16:25:42 -0400 Subject: [PATCH 28/51] Modified standalone test scripts to wait until the number of lines of PPM logs is above 100 instead of waiting a fixed amount of time for PPM readiness. --- do_kafka_test.sh | 9 +++++++++ test-scripts/standalone.sh | 11 ++++++++--- test-scripts/standalone_multi.sh | 12 +++++++++--- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 1c19dfe8..0d27a038 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -30,7 +30,10 @@ setup() { } waitForKafkaToCreateTopics() { + maxAttempts=100 + attempts=0 while true; do + attempts=$((attempts+1)) if [ $(docker ps | grep $KAFKA_CONTAINER_NAME | wc -l) == "0" ]; then echo "Kafka container '$KAFKA_CONTAINER_NAME' is not running. Exiting." ./stop_kafka.sh @@ -55,6 +58,12 @@ waitForKafkaToCreateTopics() { fi sleep 1 + + if [ $attempts -ge $maxAttempts ]; then + echo "Kafka has not created all required topics after $maxAttempts attempts. Exiting." + ./stop_kafka.sh + exit 1 + fi done } diff --git a/test-scripts/standalone.sh b/test-scripts/standalone.sh index 3e4c5a66..03d06781 100755 --- a/test-scripts/standalone.sh +++ b/test-scripts/standalone.sh @@ -17,7 +17,6 @@ PPM_CONTAINER_NAME=ppm_kafka PPM_IMAGE_TAG=do-kafka-test-ppm-image PPM_IMAGE_NAME=jpo-cvdp_ppm -SECONDS_TO_WAIT_FOR_PPM_READINESS=20 startPPMContainer() { stopPPMContainer @@ -37,8 +36,14 @@ startPPMContainer() { echo "Starting PPM in new container '$PPM_CONTAINER_NAME'" docker run --name $PPM_CONTAINER_NAME --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true -v /tmp/docker-test/data:/ppm_data -d -p '8080:8080' $PPM_IMAGE_NAME:$PPM_IMAGE_TAG /cvdi-stream/docker-test/ppm_standalone.sh - echo "Giving $PPM_CONTAINER_NAME $SECONDS_TO_WAIT_FOR_PPM_READINESS seconds to spin up" - sleep $SECONDS_TO_WAIT_FOR_PPM_READINESS + echo "Waiting for $PPM_CONTAINER_NAME to spin up" + # while num lines of docker logs is less than 100, sleep 1 + secondsWaited=0 + while [ $(docker logs $PPM_CONTAINER_NAME | wc -l) -lt 100 ]; do + sleep 1 + secondsWaited=$((secondsWaited+1)) + done + echo "$PPM_CONTAINER_NAME is ready after $secondsWaited seconds" if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) == "0" ]; then echo "PPM container '$PPM_CONTAINER_NAME' is not running. Exiting." diff --git a/test-scripts/standalone_multi.sh b/test-scripts/standalone_multi.sh index ae937498..3a2b5559 100755 --- a/test-scripts/standalone_multi.sh +++ b/test-scripts/standalone_multi.sh @@ -12,7 +12,6 @@ PPM_BSM_CONTAINER_NAME=ppm_bsm_kafka PPM_TIM_CONTAINER_NAME=ppm_tim_kafka PPM_IMAGE_TAG=do-kafka-test-ppm-image PPM_IMAGE_NAME=jpo-cvdp_ppm -SECONDS_TO_WAIT_FOR_PPM_READINESS=20 USAGE="standalone_multi.sh [MAP_FILE] [BSM_CONFIG] [TIM_CONFIG] [BSM_TEST_FILE] [TIM_TEST_FILE] [BSM_OFFSET] [TIM_OFSET]" @@ -37,8 +36,15 @@ startPPMContainer() { echo "Starting PPM in new container" docker run --name $PPM_CONTAINER_NAME --env DOCKER_HOST_IP=$dockerHostIp --env PPM_LOG_TO_CONSOLE=true --env PPM_LOG_TO_FILE=true -v $data_source:/ppm_data -d -p $ppm_container_port':8080' $PPM_IMAGE_NAME:$PPM_IMAGE_TAG /cvdi-stream/docker-test/ppm_standalone.sh - echo "Giving $PPM_CONTAINER_NAME $SECONDS_TO_WAIT_FOR_PPM_READINESS seconds to spin up" - sleep $SECONDS_TO_WAIT_FOR_PPM_READINESS + echo "Waiting for $PPM_CONTAINER_NAME to spin up" + # while num lines of docker logs is less than 100, sleep 1 + secondsWaited=0 + while [ $(docker logs $PPM_CONTAINER_NAME | wc -l) -lt 100 ]; do + sleep 1 + secondsWaited=$((secondsWaited+1)) + done + echo "$PPM_CONTAINER_NAME is ready after $secondsWaited seconds" + if [ $(docker ps | grep $PPM_CONTAINER_NAME | wc -l) == "0" ]; then echo "PPM container '$PPM_CONTAINER_NAME' is not running. Exiting." From 474c1f4a0d937df24efcfeb4d476574d63089d64 Mon Sep 17 00:00:00 2001 From: Saikrishna Bairamoni <84093461+SaikrishnaBairamoni@users.noreply.github.com> Date: Wed, 12 Jul 2023 15:51:51 -0400 Subject: [PATCH 29/51] find signal.h --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9fd6977..4063e2e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,8 @@ jobs: run: | cd $GITHUB_WORKSPACE chmod 755 -R /__w/jpo-cvdp/jpo-cvdp + cat /usr/include/asm-generic/signal.h + cat /usr/include/x86_64-linux-gnu/asm/signal.h export LD_LIBRARY_PATH=/usr/local/lib build-wrapper-linux-x86-64 --out-dir $GITHUB_WORKSPACE/bw-output ./build.sh mkdir coverage From e8e64fc1d00a234a91798b24589500e47f2ef298 Mon Sep 17 00:00:00 2001 From: Saikrishna Bairamoni <84093461+SaikrishnaBairamoni@users.noreply.github.com> Date: Wed, 12 Jul 2023 17:27:17 -0400 Subject: [PATCH 30/51] try docker buildx action --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4063e2e0..fafa22e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,11 @@ jobs: cmake --build _cmake_build cmake --build _cmake_build --target install - name: Install sonar-scanner and build-wrapper - uses: SonarSource/sonarcloud-github-c-cpp@v1 # This Action Installs sonar cloud and build wrapper to run sonar scan analysis + uses: SonarSource/sonarcloud-github-c-cpp@v1 # This Action Installs sonar cloud and build wrapper to run sonar scan analysis + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build + uses: docker/build-push-action@v3 - name: Build and Generate test coverage run: | cd $GITHUB_WORKSPACE From 9f4b90bf52910255ea2bef05e14fd9aeeea63360 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Tue, 18 Jul 2023 12:23:18 -0400 Subject: [PATCH 31/51] Modified note about WSL hanging in the README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 410491e5..dc316c8a 100644 --- a/README.md +++ b/README.md @@ -234,7 +234,7 @@ The DOCKER_HOST_IP environment variable must be set to the IP address of the hos export DOCKER_HOST_IP=$(ifconfig | zgrep -m 1 -oP '(?<=inet\s)\d+(\.\d+){3}') ``` -WSL will sometimes hang indefinitely while script waits for kafka to create topics. When this happens, running `wsl --shutdown` in a windows command prompt and restarting the docker services is recommended. +WSL will sometimes hang while the script waits for kafka to create topics. The script should exit after a number of attempts, but if it does not, running `wsl --shutdown` in a windows command prompt and restarting the docker services is recommended. ## Some Notes - The tests for this project can be run after compilation by running the "ppm_tests" executable. From 144a91f2ad0120e058e748d349c9f591e5a372dd Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Tue, 18 Jul 2023 12:54:09 -0400 Subject: [PATCH 32/51] Modified PPM container name in `do_kafka_test.sh`, `standalone.sh` & `standalone-multi.sh` --- do_kafka_test.sh | 2 +- test-scripts/standalone.sh | 2 +- test-scripts/standalone_multi.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 0d27a038..847b3bb4 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -16,7 +16,7 @@ KAFKA_CONTAINER_NAME=jpo-cvdp-kafka-1 MAP_FILE=data/I_80.edges BSM_DATA_FILE=data/I_80_test.json TIM_DATA_FILE=data/I_80_test_TIMS.json -PPM_CONTAINER_NAME=ppm_kafka +PPM_CONTAINER_NAME=test_ppm_instance PPM_IMAGE_TAG=do-kafka-test-ppm-image PPM_IMAGE_NAME=jpo-cvdp_ppm diff --git a/test-scripts/standalone.sh b/test-scripts/standalone.sh index 03d06781..d9d3371d 100755 --- a/test-scripts/standalone.sh +++ b/test-scripts/standalone.sh @@ -14,7 +14,7 @@ # This script should only be used for testing or demo purposes, as it may hang if the offsets are wrong. It also # checks if the required configuration and test data files exist before proceeding with the test. -PPM_CONTAINER_NAME=ppm_kafka +PPM_CONTAINER_NAME=test_ppm_instance PPM_IMAGE_TAG=do-kafka-test-ppm-image PPM_IMAGE_NAME=jpo-cvdp_ppm diff --git a/test-scripts/standalone_multi.sh b/test-scripts/standalone_multi.sh index 3a2b5559..7f743b28 100755 --- a/test-scripts/standalone_multi.sh +++ b/test-scripts/standalone_multi.sh @@ -8,8 +8,8 @@ # This script should only be used for testing or demo purposes, as it may hang if the offsets are wrong. It also # checks if the required configuration and test data files exist before proceeding with the test. -PPM_BSM_CONTAINER_NAME=ppm_bsm_kafka -PPM_TIM_CONTAINER_NAME=ppm_tim_kafka +PPM_BSM_CONTAINER_NAME=test_ppm_bsm_instance +PPM_TIM_CONTAINER_NAME=test_ppm_tim_instance PPM_IMAGE_TAG=do-kafka-test-ppm-image PPM_IMAGE_NAME=jpo-cvdp_ppm From 69af1ac421ab7ab71465081ae8b705c8e302a911 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Tue, 18 Jul 2023 13:23:25 -0400 Subject: [PATCH 33/51] Used name of current directory when defining kafka container name in `do_kafka_test.sh` --- do_kafka_test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 847b3bb4..89471741 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -12,7 +12,8 @@ CYAN='\033[0;36m' YELLOW='\033[1;33m' NC='\033[0m' # No Color -KAFKA_CONTAINER_NAME=jpo-cvdp-kafka-1 +CURRENT_DIR_NAME=${PWD##*/} +KAFKA_CONTAINER_NAME=$CURRENT_DIR_NAME-kafka-1 MAP_FILE=data/I_80.edges BSM_DATA_FILE=data/I_80_test.json TIM_DATA_FILE=data/I_80_test_TIMS.json From b3c12407ff808cd4282c36bca644b06eb457e5c8 Mon Sep 17 00:00:00 2001 From: Marc Wodahl Date: Tue, 18 Jul 2023 12:30:10 -0600 Subject: [PATCH 34/51] update logging level strings --- sample.env | 2 +- src/ppm.cpp | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sample.env b/sample.env index dd9d5c56..f23c2d0e 100644 --- a/sample.env +++ b/sample.env @@ -15,7 +15,7 @@ REDACTION_PROPERTIES_PATH= # Logging related flags for file/console logging & log level PPM_LOG_TO_FILE=false PPM_LOG_TO_CONSOLE=true -PPM_LOG_LEVEL=info +PPM_LOG_LEVEL=INFO # When set to true, the RPM will print debug messages to a file. (this is separate from regular logging) RPM_DEBUG= diff --git a/src/ppm.cpp b/src/ppm.cpp index c7a26c20..312e00af 100644 --- a/src/ppm.cpp +++ b/src/ppm.cpp @@ -245,19 +245,19 @@ void PPM::print_configuration() const bool PPM::configure() { if ( optIsSet('v') ) { - if ( "trace" == optString('v') ) { + if ( "TRACE" == optString('v') ) { logger->set_level( spdlog::level::trace ); - } else if ( "debug" == optString('v') ) { + } else if ( "DEBUG" == optString('v') ) { logger->set_level( spdlog::level::trace ); - } else if ( "info" == optString('v') ) { + } else if ( "INFO" == optString('v') ) { logger->set_level( spdlog::level::trace ); - } else if ( "warning" == optString('v') ) { + } else if ( "WARNING" == optString('v') ) { logger->set_level( spdlog::level::warn ); - } else if ( "error" == optString('v') ) { + } else if ( "ERROR" == optString('v') ) { logger->set_level( spdlog::level::err ); - } else if ( "critical" == optString('v') ) { + } else if ( "CRITICAL" == optString('v') ) { logger->set_level( spdlog::level::critical ); - } else if ( "off" == optString('v') ) { + } else if ( "OFF" == optString('v') ) { logger->set_level( spdlog::level::off ); } else { logger->warn("information logger level was configured but unreadable; using default."); From 080ae109f4fd9233cc67e65a941357484d1b4869 Mon Sep 17 00:00:00 2001 From: dan-du-car Date: Wed, 19 Jul 2023 15:56:05 -0400 Subject: [PATCH 35/51] update --- include/catch/catch.hpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/include/catch/catch.hpp b/include/catch/catch.hpp index a0ecd16d..4fe35867 100644 --- a/include/catch/catch.hpp +++ b/include/catch/catch.hpp @@ -6440,7 +6440,9 @@ namespace Catch { # else // CATCH_CONFIG_POSIX_SIGNALS is defined #include - +#ifndef SIGSTKSZ +#define SIGSTKSZ 32768 +#endif namespace Catch { struct SignalDefs { @@ -6462,7 +6464,7 @@ namespace Catch { static bool isSet; static struct sigaction oldSigActions [sizeof(signalDefs)/sizeof(SignalDefs)]; static stack_t oldSigStack; - static char altStackMem[32768]; + static char altStackMem[SIGSTKSZ]; static void handleSignal( int sig ) { std::string name = ""; @@ -6482,7 +6484,7 @@ namespace Catch { isSet = true; stack_t sigStack; sigStack.ss_sp = altStackMem; - sigStack.ss_size = 32768; + sigStack.ss_size = SIGSTKSZ; sigStack.ss_flags = 0; sigaltstack(&sigStack, &oldSigStack); struct sigaction sa = { 0 }; @@ -6513,7 +6515,7 @@ namespace Catch { bool FatalConditionHandler::isSet = false; struct sigaction FatalConditionHandler::oldSigActions[sizeof(signalDefs)/sizeof(SignalDefs)] = {}; stack_t FatalConditionHandler::oldSigStack = {}; - char FatalConditionHandler::altStackMem[32768] = {}; + char FatalConditionHandler::altStackMem[SIGSTKSZ] = {}; } // namespace Catch From ce5cab0edb8d787decf57c3fdcef01447a0144c1 Mon Sep 17 00:00:00 2001 From: dan-du-car Date: Wed, 19 Jul 2023 16:04:22 -0400 Subject: [PATCH 36/51] update --- include/catch/catch.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/catch/catch.hpp b/include/catch/catch.hpp index 4fe35867..3104ed81 100644 --- a/include/catch/catch.hpp +++ b/include/catch/catch.hpp @@ -6440,9 +6440,8 @@ namespace Catch { # else // CATCH_CONFIG_POSIX_SIGNALS is defined #include -#ifndef SIGSTKSZ #define SIGSTKSZ 32768 -#endif + namespace Catch { struct SignalDefs { From 639fd0bb52499ef921cc83c0e263e85f1caf60d3 Mon Sep 17 00:00:00 2001 From: dan-du-car Date: Wed, 19 Jul 2023 16:09:19 -0400 Subject: [PATCH 37/51] update catch version --- include/catch/catch.hpp | 24229 ++++++++++++++++++++++++-------------- 1 file changed, 15356 insertions(+), 8873 deletions(-) diff --git a/include/catch/catch.hpp b/include/catch/catch.hpp index 3104ed81..7e706f94 100644 --- a/include/catch/catch.hpp +++ b/include/catch/catch.hpp @@ -1,17 +1,21 @@ /* - * Catch v1.9.3 - * Generated: 2017-04-25 14:16:29.434734 + * Catch v2.13.7 + * Generated: 2021-07-28 20:29:27.753164 * ---------------------------------------------------------- * This file has been merged from multiple headers. Please don't edit it directly - * Copyright (c) 2012 Two Blue Cubes Ltd. All rights reserved. + * Copyright (c) 2021 Two Blue Cubes Ltd. All rights reserved. * * Distributed under the Boost Software License, Version 1.0. (See accompanying * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) */ #ifndef TWOBLUECUBES_SINGLE_INCLUDE_CATCH_HPP_INCLUDED #define TWOBLUECUBES_SINGLE_INCLUDE_CATCH_HPP_INCLUDED +// start catch.hpp -#define TWOBLUECUBES_CATCH_HPP_INCLUDED + +#define CATCH_VERSION_MAJOR 2 +#define CATCH_VERSION_MINOR 13 +#define CATCH_VERSION_PATCH 7 #ifdef __clang__ # pragma clang system_header @@ -19,36 +23,69 @@ # pragma GCC system_header #endif -// #included from: internal/catch_suppress_warnings.h +// start catch_suppress_warnings.h #ifdef __clang__ # ifdef __ICC // icpc defines the __clang__ macro # pragma warning(push) # pragma warning(disable: 161 1682) # else // __ICC -# pragma clang diagnostic ignored "-Wglobal-constructors" -# pragma clang diagnostic ignored "-Wvariadic-macros" -# pragma clang diagnostic ignored "-Wc99-extensions" -# pragma clang diagnostic ignored "-Wunused-variable" # pragma clang diagnostic push # pragma clang diagnostic ignored "-Wpadded" -# pragma clang diagnostic ignored "-Wc++98-compat" -# pragma clang diagnostic ignored "-Wc++98-compat-pedantic" # pragma clang diagnostic ignored "-Wswitch-enum" # pragma clang diagnostic ignored "-Wcovered-switch-default" # endif #elif defined __GNUC__ -# pragma GCC diagnostic ignored "-Wvariadic-macros" -# pragma GCC diagnostic ignored "-Wunused-variable" -# pragma GCC diagnostic ignored "-Wparentheses" + // Because REQUIREs trigger GCC's -Wparentheses, and because still + // supported version of g++ have only buggy support for _Pragmas, + // Wparentheses have to be suppressed globally. +# pragma GCC diagnostic ignored "-Wparentheses" // See #674 for details # pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-variable" # pragma GCC diagnostic ignored "-Wpadded" #endif +// end catch_suppress_warnings.h #if defined(CATCH_CONFIG_MAIN) || defined(CATCH_CONFIG_RUNNER) # define CATCH_IMPL +# define CATCH_CONFIG_ALL_PARTS +#endif + +// In the impl file, we want to have access to all parts of the headers +// Can also be used to sanely support PCHs +#if defined(CATCH_CONFIG_ALL_PARTS) +# define CATCH_CONFIG_EXTERNAL_INTERFACES +# if defined(CATCH_CONFIG_DISABLE_MATCHERS) +# undef CATCH_CONFIG_DISABLE_MATCHERS +# endif +# if !defined(CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER) +# define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER +# endif +#endif + +#if !defined(CATCH_CONFIG_IMPL_ONLY) +// start catch_platform.h + +// See e.g.: +// https://opensource.apple.com/source/CarbonHeaders/CarbonHeaders-18.1/TargetConditionals.h.auto.html +#ifdef __APPLE__ +# include +# if (defined(TARGET_OS_OSX) && TARGET_OS_OSX == 1) || \ + (defined(TARGET_OS_MAC) && TARGET_OS_MAC == 1) +# define CATCH_PLATFORM_MAC +# elif (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE == 1) +# define CATCH_PLATFORM_IPHONE +# endif + +#elif defined(linux) || defined(__linux) || defined(__linux__) +# define CATCH_PLATFORM_LINUX + +#elif defined(WIN32) || defined(__WIN32__) || defined(_WIN32) || defined(_MSC_VER) || defined(__MINGW32__) +# define CATCH_PLATFORM_WINDOWS #endif +// end catch_platform.h + #ifdef CATCH_IMPL # ifndef CLARA_CONFIG_MAIN # define CLARA_CONFIG_MAIN_NOT_DEFINED @@ -56,93 +93,130 @@ # endif #endif -// #included from: internal/catch_notimplemented_exception.h -#define TWOBLUECUBES_CATCH_NOTIMPLEMENTED_EXCEPTION_H_INCLUDED +// start catch_user_interfaces.h -// #included from: catch_common.h -#define TWOBLUECUBES_CATCH_COMMON_H_INCLUDED +namespace Catch { + unsigned int rngSeed(); +} + +// end catch_user_interfaces.h +// start catch_tag_alias_autoregistrar.h -// #included from: catch_compiler_capabilities.h -#define TWOBLUECUBES_CATCH_COMPILER_CAPABILITIES_HPP_INCLUDED +// start catch_common.h -// Detect a number of compiler features - mostly C++11/14 conformance - by compiler +// start catch_compiler_capabilities.h + +// Detect a number of compiler features - by compiler // The following features are defined: // -// CATCH_CONFIG_CPP11_NULLPTR : is nullptr supported? -// CATCH_CONFIG_CPP11_NOEXCEPT : is noexcept supported? -// CATCH_CONFIG_CPP11_GENERATED_METHODS : The delete and default keywords for compiler generated methods -// CATCH_CONFIG_CPP11_IS_ENUM : std::is_enum is supported? -// CATCH_CONFIG_CPP11_TUPLE : std::tuple is supported -// CATCH_CONFIG_CPP11_LONG_LONG : is long long supported? -// CATCH_CONFIG_CPP11_OVERRIDE : is override supported? -// CATCH_CONFIG_CPP11_UNIQUE_PTR : is unique_ptr supported (otherwise use auto_ptr) -// CATCH_CONFIG_CPP11_SHUFFLE : is std::shuffle supported? -// CATCH_CONFIG_CPP11_TYPE_TRAITS : are type_traits and enable_if supported? - -// CATCH_CONFIG_CPP11_OR_GREATER : Is C++11 supported? - -// CATCH_CONFIG_VARIADIC_MACROS : are variadic macros supported? // CATCH_CONFIG_COUNTER : is the __COUNTER__ macro supported? // CATCH_CONFIG_WINDOWS_SEH : is Windows SEH supported? // CATCH_CONFIG_POSIX_SIGNALS : are POSIX signals supported? +// CATCH_CONFIG_DISABLE_EXCEPTIONS : Are exceptions enabled? // **************** // Note to maintainers: if new toggles are added please document them // in configuration.md, too // **************** // In general each macro has a _NO_ form -// (e.g. CATCH_CONFIG_CPP11_NO_NULLPTR) which disables the feature. +// (e.g. CATCH_CONFIG_NO_POSIX_SIGNALS) which disables the feature. // Many features, at point of detection, define an _INTERNAL_ macro, so they // can be combined, en-mass, with the _NO_ forms later. -// All the C++11 features can be disabled with CATCH_CONFIG_NO_CPP11 - #ifdef __cplusplus -# if __cplusplus >= 201103L -# define CATCH_CPP11_OR_GREATER +# if (__cplusplus >= 201402L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201402L) +# define CATCH_CPP14_OR_GREATER # endif -# if __cplusplus >= 201402L -# define CATCH_CPP14_OR_GREATER +# if (__cplusplus >= 201703L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) +# define CATCH_CPP17_OR_GREATER # endif #endif -#ifdef __clang__ +// Only GCC compiler should be used in this block, so other compilers trying to +// mask themselves as GCC should be ignored. +#if defined(__GNUC__) && !defined(__clang__) && !defined(__ICC) && !defined(__CUDACC__) && !defined(__LCC__) +# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION _Pragma( "GCC diagnostic push" ) +# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION _Pragma( "GCC diagnostic pop" ) -# if __has_feature(cxx_nullptr) -# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR -# endif +# define CATCH_INTERNAL_IGNORE_BUT_WARN(...) (void)__builtin_constant_p(__VA_ARGS__) + +#endif -# if __has_feature(cxx_noexcept) -# define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT +#if defined(__clang__) + +# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION _Pragma( "clang diagnostic push" ) +# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION _Pragma( "clang diagnostic pop" ) + +// As of this writing, IBM XL's implementation of __builtin_constant_p has a bug +// which results in calls to destructors being emitted for each temporary, +// without a matching initialization. In practice, this can result in something +// like `std::string::~string` being called on an uninitialized value. +// +// For example, this code will likely segfault under IBM XL: +// ``` +// REQUIRE(std::string("12") + "34" == "1234") +// ``` +// +// Therefore, `CATCH_INTERNAL_IGNORE_BUT_WARN` is not implemented. +# if !defined(__ibmxl__) && !defined(__CUDACC__) +# define CATCH_INTERNAL_IGNORE_BUT_WARN(...) (void)__builtin_constant_p(__VA_ARGS__) /* NOLINT(cppcoreguidelines-pro-type-vararg, hicpp-vararg) */ # endif -# if defined(CATCH_CPP11_OR_GREATER) -# define CATCH_INTERNAL_SUPPRESS_ETD_WARNINGS \ - _Pragma( "clang diagnostic push" ) \ - _Pragma( "clang diagnostic ignored \"-Wexit-time-destructors\"" ) -# define CATCH_INTERNAL_UNSUPPRESS_ETD_WARNINGS \ - _Pragma( "clang diagnostic pop" ) - -# define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS \ - _Pragma( "clang diagnostic push" ) \ - _Pragma( "clang diagnostic ignored \"-Wparentheses\"" ) -# define CATCH_INTERNAL_UNSUPPRESS_PARENTHESES_WARNINGS \ - _Pragma( "clang diagnostic pop" ) -# endif +# define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wexit-time-destructors\"" ) \ + _Pragma( "clang diagnostic ignored \"-Wglobal-constructors\"") + +# define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wparentheses\"" ) + +# define CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wunused-variable\"" ) + +# define CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wgnu-zero-variadic-macro-arguments\"" ) + +# define CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wunused-template\"" ) #endif // __clang__ +//////////////////////////////////////////////////////////////////////////////// +// Assume that non-Windows platforms support posix signals by default +#if !defined(CATCH_PLATFORM_WINDOWS) + #define CATCH_INTERNAL_CONFIG_POSIX_SIGNALS +#endif + //////////////////////////////////////////////////////////////////////////////// // We know some environments not to support full POSIX signals -#if defined(__CYGWIN__) || defined(__QNX__) +#if defined(__CYGWIN__) || defined(__QNX__) || defined(__EMSCRIPTEN__) || defined(__DJGPP__) + #define CATCH_INTERNAL_CONFIG_NO_POSIX_SIGNALS +#endif -# if !defined(CATCH_CONFIG_POSIX_SIGNALS) +#ifdef __OS400__ # define CATCH_INTERNAL_CONFIG_NO_POSIX_SIGNALS -# endif +# define CATCH_CONFIG_COLOUR_NONE +#endif + +//////////////////////////////////////////////////////////////////////////////// +// Android somehow still does not support std::to_string +#if defined(__ANDROID__) +# define CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING +# define CATCH_INTERNAL_CONFIG_ANDROID_LOGWRITE +#endif + +//////////////////////////////////////////////////////////////////////////////// +// Not all Windows environments support SEH properly +#if defined(__MINGW32__) +# define CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH +#endif +//////////////////////////////////////////////////////////////////////////////// +// PS4 +#if defined(__ORBIS__) +# define CATCH_INTERNAL_CONFIG_NO_NEW_CAPTURE #endif //////////////////////////////////////////////////////////////////////////////// @@ -152,217 +226,243 @@ // Required for some versions of Cygwin to declare gettimeofday // see: http://stackoverflow.com/questions/36901803/gettimeofday-not-declared-in-this-scope-cygwin # define _BSD_SOURCE +// some versions of cygwin (most) do not support std::to_string. Use the libstd check. +// https://gcc.gnu.org/onlinedocs/gcc-4.8.2/libstdc++/api/a01053_source.html line 2812-2813 +# if !((__cplusplus >= 201103L) && defined(_GLIBCXX_USE_C99) \ + && !defined(_GLIBCXX_HAVE_BROKEN_VSWPRINTF)) -#endif // __CYGWIN__ - -//////////////////////////////////////////////////////////////////////////////// -// Borland -#ifdef __BORLANDC__ - -#endif // __BORLANDC__ - -//////////////////////////////////////////////////////////////////////////////// -// EDG -#ifdef __EDG_VERSION__ +# define CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING -#endif // __EDG_VERSION__ +# endif +#endif // __CYGWIN__ //////////////////////////////////////////////////////////////////////////////// -// Digital Mars -#ifdef __DMC__ +// Visual C++ +#if defined(_MSC_VER) -#endif // __DMC__ +# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION __pragma( warning(push) ) +# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION __pragma( warning(pop) ) -//////////////////////////////////////////////////////////////////////////////// -// GCC -#ifdef __GNUC__ +// Universal Windows platform does not support SEH +// Or console colours (or console at all...) +# if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP) +# define CATCH_CONFIG_COLOUR_NONE +# else +# define CATCH_INTERNAL_CONFIG_WINDOWS_SEH +# endif -# if __GNUC__ == 4 && __GNUC_MINOR__ >= 6 && defined(__GXX_EXPERIMENTAL_CXX0X__) -# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR -# endif +// MSVC traditional preprocessor needs some workaround for __VA_ARGS__ +// _MSVC_TRADITIONAL == 0 means new conformant preprocessor +// _MSVC_TRADITIONAL == 1 means old traditional non-conformant preprocessor +# if !defined(__clang__) // Handle Clang masquerading for msvc +# if !defined(_MSVC_TRADITIONAL) || (defined(_MSVC_TRADITIONAL) && _MSVC_TRADITIONAL) +# define CATCH_INTERNAL_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +# endif // MSVC_TRADITIONAL +# endif // __clang__ -// - otherwise more recent versions define __cplusplus >= 201103L -// and will get picked up below +#endif // _MSC_VER -#endif // __GNUC__ +#if defined(_REENTRANT) || defined(_MSC_VER) +// Enable async processing, as -pthread is specified or no additional linking is required +# define CATCH_INTERNAL_CONFIG_USE_ASYNC +#endif // _MSC_VER //////////////////////////////////////////////////////////////////////////////// -// Visual C++ -#ifdef _MSC_VER - -#define CATCH_INTERNAL_CONFIG_WINDOWS_SEH - -#if (_MSC_VER >= 1600) -# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR -# define CATCH_INTERNAL_CONFIG_CPP11_UNIQUE_PTR -#endif - -#if (_MSC_VER >= 1900 ) // (VC++ 13 (VS2015)) -#define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT -#define CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS -#define CATCH_INTERNAL_CONFIG_CPP11_SHUFFLE -#define CATCH_INTERNAL_CONFIG_CPP11_TYPE_TRAITS +// Check if we are compiled with -fno-exceptions or equivalent +#if defined(__EXCEPTIONS) || defined(__cpp_exceptions) || defined(_CPPUNWIND) +# define CATCH_INTERNAL_CONFIG_EXCEPTIONS_ENABLED #endif -#endif // _MSC_VER - //////////////////////////////////////////////////////////////////////////////// +// DJGPP +#ifdef __DJGPP__ +# define CATCH_INTERNAL_CONFIG_NO_WCHAR +#endif // __DJGPP__ -// Use variadic macros if the compiler supports them -#if ( defined _MSC_VER && _MSC_VER > 1400 && !defined __EDGE__) || \ - ( defined __WAVE__ && __WAVE_HAS_VARIADICS ) || \ - ( defined __GNUC__ && __GNUC__ >= 3 ) || \ - ( !defined __cplusplus && __STDC_VERSION__ >= 199901L || __cplusplus >= 201103L ) - -#define CATCH_INTERNAL_CONFIG_VARIADIC_MACROS - +//////////////////////////////////////////////////////////////////////////////// +// Embarcadero C++Build +#if defined(__BORLANDC__) + #define CATCH_INTERNAL_CONFIG_POLYFILL_ISNAN #endif -// Use __COUNTER__ if the compiler supports it -#if ( defined _MSC_VER && _MSC_VER >= 1300 ) || \ - ( defined __GNUC__ && __GNUC__ >= 4 && __GNUC_MINOR__ >= 3 ) || \ - ( defined __clang__ && __clang_major__ >= 3 ) - -#define CATCH_INTERNAL_CONFIG_COUNTER +//////////////////////////////////////////////////////////////////////////////// +// Use of __COUNTER__ is suppressed during code analysis in +// CLion/AppCode 2017.2.x and former, because __COUNTER__ is not properly +// handled by it. +// Otherwise all supported compilers support COUNTER macro, +// but user still might want to turn it off +#if ( !defined(__JETBRAINS_IDE__) || __JETBRAINS_IDE__ >= 20170300L ) + #define CATCH_INTERNAL_CONFIG_COUNTER #endif //////////////////////////////////////////////////////////////////////////////// -// C++ language feature support - -// catch all support for C++11 -#if defined(CATCH_CPP11_OR_GREATER) - -# if !defined(CATCH_INTERNAL_CONFIG_CPP11_NULLPTR) -# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR -# endif -# ifndef CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT -# define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT -# endif +// RTX is a special version of Windows that is real time. +// This means that it is detected as Windows, but does not provide +// the same set of capabilities as real Windows does. +#if defined(UNDER_RTSS) || defined(RTX64_BUILD) + #define CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH + #define CATCH_INTERNAL_CONFIG_NO_ASYNC + #define CATCH_CONFIG_COLOUR_NONE +#endif -# ifndef CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS -# define CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS -# endif +#if !defined(_GLIBCXX_USE_C99_MATH_TR1) +#define CATCH_INTERNAL_CONFIG_GLOBAL_NEXTAFTER +#endif -# ifndef CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM -# define CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM -# endif +// Various stdlib support checks that require __has_include +#if defined(__has_include) + // Check if string_view is available and usable + #if __has_include() && defined(CATCH_CPP17_OR_GREATER) + # define CATCH_INTERNAL_CONFIG_CPP17_STRING_VIEW + #endif + + // Check if optional is available and usable + # if __has_include() && defined(CATCH_CPP17_OR_GREATER) + # define CATCH_INTERNAL_CONFIG_CPP17_OPTIONAL + # endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) + + // Check if byte is available and usable + # if __has_include() && defined(CATCH_CPP17_OR_GREATER) + # include + # if defined(__cpp_lib_byte) && (__cpp_lib_byte > 0) + # define CATCH_INTERNAL_CONFIG_CPP17_BYTE + # endif + # endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) + + // Check if variant is available and usable + # if __has_include() && defined(CATCH_CPP17_OR_GREATER) + # if defined(__clang__) && (__clang_major__ < 8) + // work around clang bug with libstdc++ https://bugs.llvm.org/show_bug.cgi?id=31852 + // fix should be in clang 8, workaround in libstdc++ 8.2 + # include + # if defined(__GLIBCXX__) && defined(_GLIBCXX_RELEASE) && (_GLIBCXX_RELEASE < 9) + # define CATCH_CONFIG_NO_CPP17_VARIANT + # else + # define CATCH_INTERNAL_CONFIG_CPP17_VARIANT + # endif // defined(__GLIBCXX__) && defined(_GLIBCXX_RELEASE) && (_GLIBCXX_RELEASE < 9) + # else + # define CATCH_INTERNAL_CONFIG_CPP17_VARIANT + # endif // defined(__clang__) && (__clang_major__ < 8) + # endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) +#endif // defined(__has_include) + +#if defined(CATCH_INTERNAL_CONFIG_COUNTER) && !defined(CATCH_CONFIG_NO_COUNTER) && !defined(CATCH_CONFIG_COUNTER) +# define CATCH_CONFIG_COUNTER +#endif +#if defined(CATCH_INTERNAL_CONFIG_WINDOWS_SEH) && !defined(CATCH_CONFIG_NO_WINDOWS_SEH) && !defined(CATCH_CONFIG_WINDOWS_SEH) && !defined(CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH) +# define CATCH_CONFIG_WINDOWS_SEH +#endif +// This is set by default, because we assume that unix compilers are posix-signal-compatible by default. +#if defined(CATCH_INTERNAL_CONFIG_POSIX_SIGNALS) && !defined(CATCH_INTERNAL_CONFIG_NO_POSIX_SIGNALS) && !defined(CATCH_CONFIG_NO_POSIX_SIGNALS) && !defined(CATCH_CONFIG_POSIX_SIGNALS) +# define CATCH_CONFIG_POSIX_SIGNALS +#endif +// This is set by default, because we assume that compilers with no wchar_t support are just rare exceptions. +#if !defined(CATCH_INTERNAL_CONFIG_NO_WCHAR) && !defined(CATCH_CONFIG_NO_WCHAR) && !defined(CATCH_CONFIG_WCHAR) +# define CATCH_CONFIG_WCHAR +#endif -# ifndef CATCH_INTERNAL_CONFIG_CPP11_TUPLE -# define CATCH_INTERNAL_CONFIG_CPP11_TUPLE -# endif +#if !defined(CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING) && !defined(CATCH_CONFIG_NO_CPP11_TO_STRING) && !defined(CATCH_CONFIG_CPP11_TO_STRING) +# define CATCH_CONFIG_CPP11_TO_STRING +#endif -# ifndef CATCH_INTERNAL_CONFIG_VARIADIC_MACROS -# define CATCH_INTERNAL_CONFIG_VARIADIC_MACROS -# endif +#if defined(CATCH_INTERNAL_CONFIG_CPP17_OPTIONAL) && !defined(CATCH_CONFIG_NO_CPP17_OPTIONAL) && !defined(CATCH_CONFIG_CPP17_OPTIONAL) +# define CATCH_CONFIG_CPP17_OPTIONAL +#endif -# if !defined(CATCH_INTERNAL_CONFIG_CPP11_LONG_LONG) -# define CATCH_INTERNAL_CONFIG_CPP11_LONG_LONG -# endif +#if defined(CATCH_INTERNAL_CONFIG_CPP17_STRING_VIEW) && !defined(CATCH_CONFIG_NO_CPP17_STRING_VIEW) && !defined(CATCH_CONFIG_CPP17_STRING_VIEW) +# define CATCH_CONFIG_CPP17_STRING_VIEW +#endif -# if !defined(CATCH_INTERNAL_CONFIG_CPP11_OVERRIDE) -# define CATCH_INTERNAL_CONFIG_CPP11_OVERRIDE -# endif -# if !defined(CATCH_INTERNAL_CONFIG_CPP11_UNIQUE_PTR) -# define CATCH_INTERNAL_CONFIG_CPP11_UNIQUE_PTR -# endif -# if !defined(CATCH_INTERNAL_CONFIG_CPP11_SHUFFLE) -# define CATCH_INTERNAL_CONFIG_CPP11_SHUFFLE -# endif -# if !defined(CATCH_INTERNAL_CONFIG_CPP11_TYPE_TRAITS) -# define CATCH_INTERNAL_CONFIG_CPP11_TYPE_TRAITS -# endif +#if defined(CATCH_INTERNAL_CONFIG_CPP17_VARIANT) && !defined(CATCH_CONFIG_NO_CPP17_VARIANT) && !defined(CATCH_CONFIG_CPP17_VARIANT) +# define CATCH_CONFIG_CPP17_VARIANT +#endif -#endif // __cplusplus >= 201103L +#if defined(CATCH_INTERNAL_CONFIG_CPP17_BYTE) && !defined(CATCH_CONFIG_NO_CPP17_BYTE) && !defined(CATCH_CONFIG_CPP17_BYTE) +# define CATCH_CONFIG_CPP17_BYTE +#endif -// Now set the actual defines based on the above + anything the user has configured -#if defined(CATCH_INTERNAL_CONFIG_CPP11_NULLPTR) && !defined(CATCH_CONFIG_CPP11_NO_NULLPTR) && !defined(CATCH_CONFIG_CPP11_NULLPTR) && !defined(CATCH_CONFIG_NO_CPP11) -# define CATCH_CONFIG_CPP11_NULLPTR +#if defined(CATCH_CONFIG_EXPERIMENTAL_REDIRECT) +# define CATCH_INTERNAL_CONFIG_NEW_CAPTURE #endif -#if defined(CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT) && !defined(CATCH_CONFIG_CPP11_NO_NOEXCEPT) && !defined(CATCH_CONFIG_CPP11_NOEXCEPT) && !defined(CATCH_CONFIG_NO_CPP11) -# define CATCH_CONFIG_CPP11_NOEXCEPT + +#if defined(CATCH_INTERNAL_CONFIG_NEW_CAPTURE) && !defined(CATCH_INTERNAL_CONFIG_NO_NEW_CAPTURE) && !defined(CATCH_CONFIG_NO_NEW_CAPTURE) && !defined(CATCH_CONFIG_NEW_CAPTURE) +# define CATCH_CONFIG_NEW_CAPTURE #endif -#if defined(CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS) && !defined(CATCH_CONFIG_CPP11_NO_GENERATED_METHODS) && !defined(CATCH_CONFIG_CPP11_GENERATED_METHODS) && !defined(CATCH_CONFIG_NO_CPP11) -# define CATCH_CONFIG_CPP11_GENERATED_METHODS + +#if !defined(CATCH_INTERNAL_CONFIG_EXCEPTIONS_ENABLED) && !defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) +# define CATCH_CONFIG_DISABLE_EXCEPTIONS #endif -#if defined(CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM) && !defined(CATCH_CONFIG_CPP11_NO_IS_ENUM) && !defined(CATCH_CONFIG_CPP11_IS_ENUM) && !defined(CATCH_CONFIG_NO_CPP11) -# define CATCH_CONFIG_CPP11_IS_ENUM + +#if defined(CATCH_INTERNAL_CONFIG_POLYFILL_ISNAN) && !defined(CATCH_CONFIG_NO_POLYFILL_ISNAN) && !defined(CATCH_CONFIG_POLYFILL_ISNAN) +# define CATCH_CONFIG_POLYFILL_ISNAN #endif -#if defined(CATCH_INTERNAL_CONFIG_CPP11_TUPLE) && !defined(CATCH_CONFIG_CPP11_NO_TUPLE) && !defined(CATCH_CONFIG_CPP11_TUPLE) && !defined(CATCH_CONFIG_NO_CPP11) -# define CATCH_CONFIG_CPP11_TUPLE + +#if defined(CATCH_INTERNAL_CONFIG_USE_ASYNC) && !defined(CATCH_INTERNAL_CONFIG_NO_ASYNC) && !defined(CATCH_CONFIG_NO_USE_ASYNC) && !defined(CATCH_CONFIG_USE_ASYNC) +# define CATCH_CONFIG_USE_ASYNC #endif -#if defined(CATCH_INTERNAL_CONFIG_VARIADIC_MACROS) && !defined(CATCH_CONFIG_NO_VARIADIC_MACROS) && !defined(CATCH_CONFIG_VARIADIC_MACROS) -# define CATCH_CONFIG_VARIADIC_MACROS + +#if defined(CATCH_INTERNAL_CONFIG_ANDROID_LOGWRITE) && !defined(CATCH_CONFIG_NO_ANDROID_LOGWRITE) && !defined(CATCH_CONFIG_ANDROID_LOGWRITE) +# define CATCH_CONFIG_ANDROID_LOGWRITE #endif -#if defined(CATCH_INTERNAL_CONFIG_CPP11_LONG_LONG) && !defined(CATCH_CONFIG_CPP11_NO_LONG_LONG) && !defined(CATCH_CONFIG_CPP11_LONG_LONG) && !defined(CATCH_CONFIG_NO_CPP11) -# define CATCH_CONFIG_CPP11_LONG_LONG + +#if defined(CATCH_INTERNAL_CONFIG_GLOBAL_NEXTAFTER) && !defined(CATCH_CONFIG_NO_GLOBAL_NEXTAFTER) && !defined(CATCH_CONFIG_GLOBAL_NEXTAFTER) +# define CATCH_CONFIG_GLOBAL_NEXTAFTER #endif -#if defined(CATCH_INTERNAL_CONFIG_CPP11_OVERRIDE) && !defined(CATCH_CONFIG_CPP11_NO_OVERRIDE) && !defined(CATCH_CONFIG_CPP11_OVERRIDE) && !defined(CATCH_CONFIG_NO_CPP11) -# define CATCH_CONFIG_CPP11_OVERRIDE + +// Even if we do not think the compiler has that warning, we still have +// to provide a macro that can be used by the code. +#if !defined(CATCH_INTERNAL_START_WARNINGS_SUPPRESSION) +# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION #endif -#if defined(CATCH_INTERNAL_CONFIG_CPP11_UNIQUE_PTR) && !defined(CATCH_CONFIG_CPP11_NO_UNIQUE_PTR) && !defined(CATCH_CONFIG_CPP11_UNIQUE_PTR) && !defined(CATCH_CONFIG_NO_CPP11) -# define CATCH_CONFIG_CPP11_UNIQUE_PTR +#if !defined(CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION) +# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION #endif -// Use of __COUNTER__ is suppressed if __JETBRAINS_IDE__ is #defined (meaning we're being parsed by a JetBrains IDE for -// analytics) because, at time of writing, __COUNTER__ is not properly handled by it. -// This does not affect compilation -#if defined(CATCH_INTERNAL_CONFIG_COUNTER) && !defined(CATCH_CONFIG_NO_COUNTER) && !defined(CATCH_CONFIG_COUNTER) && !defined(__JETBRAINS_IDE__) -# define CATCH_CONFIG_COUNTER +#if !defined(CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS #endif -#if defined(CATCH_INTERNAL_CONFIG_CPP11_SHUFFLE) && !defined(CATCH_CONFIG_CPP11_NO_SHUFFLE) && !defined(CATCH_CONFIG_CPP11_SHUFFLE) && !defined(CATCH_CONFIG_NO_CPP11) -# define CATCH_CONFIG_CPP11_SHUFFLE +#if !defined(CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS #endif -# if defined(CATCH_INTERNAL_CONFIG_CPP11_TYPE_TRAITS) && !defined(CATCH_CONFIG_CPP11_NO_TYPE_TRAITS) && !defined(CATCH_CONFIG_CPP11_TYPE_TRAITS) && !defined(CATCH_CONFIG_NO_CPP11) -# define CATCH_CONFIG_CPP11_TYPE_TRAITS -# endif -#if defined(CATCH_INTERNAL_CONFIG_WINDOWS_SEH) && !defined(CATCH_CONFIG_NO_WINDOWS_SEH) && !defined(CATCH_CONFIG_WINDOWS_SEH) -# define CATCH_CONFIG_WINDOWS_SEH +#if !defined(CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS #endif -// This is set by default, because we assume that unix compilers are posix-signal-compatible by default. -#if !defined(CATCH_INTERNAL_CONFIG_NO_POSIX_SIGNALS) && !defined(CATCH_CONFIG_NO_POSIX_SIGNALS) && !defined(CATCH_CONFIG_POSIX_SIGNALS) -# define CATCH_CONFIG_POSIX_SIGNALS +#if !defined(CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS #endif -#if !defined(CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS) -# define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS -# define CATCH_INTERNAL_UNSUPPRESS_PARENTHESES_WARNINGS -#endif -#if !defined(CATCH_INTERNAL_SUPPRESS_ETD_WARNINGS) -# define CATCH_INTERNAL_SUPPRESS_ETD_WARNINGS -# define CATCH_INTERNAL_UNSUPPRESS_ETD_WARNINGS +// The goal of this macro is to avoid evaluation of the arguments, but +// still have the compiler warn on problems inside... +#if !defined(CATCH_INTERNAL_IGNORE_BUT_WARN) +# define CATCH_INTERNAL_IGNORE_BUT_WARN(...) #endif -// noexcept support: -#if defined(CATCH_CONFIG_CPP11_NOEXCEPT) && !defined(CATCH_NOEXCEPT) -# define CATCH_NOEXCEPT noexcept -# define CATCH_NOEXCEPT_IS(x) noexcept(x) -#else -# define CATCH_NOEXCEPT throw() -# define CATCH_NOEXCEPT_IS(x) +#if defined(__APPLE__) && defined(__apple_build_version__) && (__clang_major__ < 10) +# undef CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS +#elif defined(__clang__) && (__clang_major__ < 5) +# undef CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS #endif -// nullptr support -#ifdef CATCH_CONFIG_CPP11_NULLPTR -# define CATCH_NULL nullptr -#else -# define CATCH_NULL NULL +#if !defined(CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS #endif -// override support -#ifdef CATCH_CONFIG_CPP11_OVERRIDE -# define CATCH_OVERRIDE override +#if defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) +#define CATCH_TRY if ((true)) +#define CATCH_CATCH_ALL if ((false)) +#define CATCH_CATCH_ANON(type) if ((false)) #else -# define CATCH_OVERRIDE +#define CATCH_TRY try +#define CATCH_CATCH_ALL catch (...) +#define CATCH_CATCH_ANON(type) catch (type) #endif -// unique_ptr support -#ifdef CATCH_CONFIG_CPP11_UNIQUE_PTR -# define CATCH_AUTO_PTR( T ) std::unique_ptr -#else -# define CATCH_AUTO_PTR( T ) std::auto_ptr +#if defined(CATCH_INTERNAL_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR) && !defined(CATCH_CONFIG_NO_TRADITIONAL_MSVC_PREPROCESSOR) && !defined(CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR) +#define CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #endif +// end catch_compiler_capabilities.h #define INTERNAL_CATCH_UNIQUE_NAME_LINE2( name, line ) name##line #define INTERNAL_CATCH_UNIQUE_NAME_LINE( name, line ) INTERNAL_CATCH_UNIQUE_NAME_LINE2( name, line ) #ifdef CATCH_CONFIG_COUNTER @@ -371,95 +471,48 @@ # define INTERNAL_CATCH_UNIQUE_NAME( name ) INTERNAL_CATCH_UNIQUE_NAME_LINE( name, __LINE__ ) #endif -#define INTERNAL_CATCH_STRINGIFY2( expr ) #expr -#define INTERNAL_CATCH_STRINGIFY( expr ) INTERNAL_CATCH_STRINGIFY2( expr ) +#include +#include +#include -#include -#include +// We need a dummy global operator<< so we can bring it into Catch namespace later +struct Catch_global_namespace_dummy {}; +std::ostream& operator<<(std::ostream&, Catch_global_namespace_dummy); namespace Catch { - struct IConfig; - struct CaseSensitive { enum Choice { Yes, No }; }; class NonCopyable { -#ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS NonCopyable( NonCopyable const& ) = delete; NonCopyable( NonCopyable && ) = delete; NonCopyable& operator = ( NonCopyable const& ) = delete; NonCopyable& operator = ( NonCopyable && ) = delete; -#else - NonCopyable( NonCopyable const& info ); - NonCopyable& operator = ( NonCopyable const& ); -#endif protected: - NonCopyable() {} + NonCopyable(); virtual ~NonCopyable(); }; - class SafeBool { - public: - typedef void (SafeBool::*type)() const; - - static type makeSafe( bool value ) { - return value ? &SafeBool::trueValue : 0; - } - private: - void trueValue() const {} - }; - - template - inline void deleteAll( ContainerT& container ) { - typename ContainerT::const_iterator it = container.begin(); - typename ContainerT::const_iterator itEnd = container.end(); - for(; it != itEnd; ++it ) - delete *it; - } - template - inline void deleteAllValues( AssociativeContainerT& container ) { - typename AssociativeContainerT::const_iterator it = container.begin(); - typename AssociativeContainerT::const_iterator itEnd = container.end(); - for(; it != itEnd; ++it ) - delete it->second; - } - - bool startsWith( std::string const& s, std::string const& prefix ); - bool startsWith( std::string const& s, char prefix ); - bool endsWith( std::string const& s, std::string const& suffix ); - bool endsWith( std::string const& s, char suffix ); - bool contains( std::string const& s, std::string const& infix ); - void toLowerInPlace( std::string& s ); - std::string toLower( std::string const& s ); - std::string trim( std::string const& str ); - bool replaceInPlace( std::string& str, std::string const& replaceThis, std::string const& withThis ); - - struct pluralise { - pluralise( std::size_t count, std::string const& label ); - - friend std::ostream& operator << ( std::ostream& os, pluralise const& pluraliser ); + struct SourceLineInfo { - std::size_t m_count; - std::string m_label; - }; + SourceLineInfo() = delete; + SourceLineInfo( char const* _file, std::size_t _line ) noexcept + : file( _file ), + line( _line ) + {} - struct SourceLineInfo { + SourceLineInfo( SourceLineInfo const& other ) = default; + SourceLineInfo& operator = ( SourceLineInfo const& ) = default; + SourceLineInfo( SourceLineInfo&& ) noexcept = default; + SourceLineInfo& operator = ( SourceLineInfo&& ) noexcept = default; - SourceLineInfo(); - SourceLineInfo( char const* _file, std::size_t _line ); -# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS - SourceLineInfo(SourceLineInfo const& other) = default; - SourceLineInfo( SourceLineInfo && ) = default; - SourceLineInfo& operator = ( SourceLineInfo const& ) = default; - SourceLineInfo& operator = ( SourceLineInfo && ) = default; -# endif - bool empty() const; - bool operator == ( SourceLineInfo const& other ) const; - bool operator < ( SourceLineInfo const& other ) const; + bool empty() const noexcept { return file[0] == '\0'; } + bool operator == ( SourceLineInfo const& other ) const noexcept; + bool operator < ( SourceLineInfo const& other ) const noexcept; char const* file; std::size_t line; @@ -467,24 +520,17 @@ namespace Catch { std::ostream& operator << ( std::ostream& os, SourceLineInfo const& info ); - // This is just here to avoid compiler warnings with macro constants and boolean literals - inline bool isTrue( bool value ){ return value; } - inline bool alwaysTrue() { return true; } - inline bool alwaysFalse() { return false; } - - void throwLogicError( std::string const& message, SourceLineInfo const& locationInfo ); - - void seedRng( IConfig const& config ); - unsigned int rngSeed(); + // Bring in operator<< from global namespace into Catch namespace + // This is necessary because the overload of operator<< above makes + // lookup stop at namespace Catch + using ::operator<<; // Use this in variadic streaming macros to allow // >> +StreamEndStop // as well as // >> stuff +StreamEndStop struct StreamEndStop { - std::string operator+() { - return std::string(); - } + std::string operator+() const; }; template T const& operator + ( T const& value, StreamEndStop ) { @@ -492,364 +538,812 @@ namespace Catch { } } -#define CATCH_INTERNAL_LINEINFO ::Catch::SourceLineInfo( __FILE__, static_cast( __LINE__ ) ) -#define CATCH_INTERNAL_ERROR( msg ) ::Catch::throwLogicError( msg, CATCH_INTERNAL_LINEINFO ); +#define CATCH_INTERNAL_LINEINFO \ + ::Catch::SourceLineInfo( __FILE__, static_cast( __LINE__ ) ) +// end catch_common.h namespace Catch { - class NotImplementedException : public std::exception - { - public: - NotImplementedException( SourceLineInfo const& lineInfo ); - NotImplementedException( NotImplementedException const& ) {} - - virtual ~NotImplementedException() CATCH_NOEXCEPT {} - - virtual const char* what() const CATCH_NOEXCEPT; - - private: - std::string m_what; - SourceLineInfo m_lineInfo; + struct RegistrarForTagAliases { + RegistrarForTagAliases( char const* alias, char const* tag, SourceLineInfo const& lineInfo ); }; } // end namespace Catch -/////////////////////////////////////////////////////////////////////////////// -#define CATCH_NOT_IMPLEMENTED throw Catch::NotImplementedException( CATCH_INTERNAL_LINEINFO ) +#define CATCH_REGISTER_TAG_ALIAS( alias, spec ) \ + CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + namespace{ Catch::RegistrarForTagAliases INTERNAL_CATCH_UNIQUE_NAME( AutoRegisterTagAlias )( alias, spec, CATCH_INTERNAL_LINEINFO ); } \ + CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION -// #included from: internal/catch_context.h -#define TWOBLUECUBES_CATCH_CONTEXT_H_INCLUDED +// end catch_tag_alias_autoregistrar.h +// start catch_test_registry.h -// #included from: catch_interfaces_generators.h -#define TWOBLUECUBES_CATCH_INTERFACES_GENERATORS_H_INCLUDED +// start catch_interfaces_testcase.h -#include +#include namespace Catch { - struct IGeneratorInfo { - virtual ~IGeneratorInfo(); - virtual bool moveNext() = 0; - virtual std::size_t getCurrentIndex() const = 0; + class TestSpec; + + struct ITestInvoker { + virtual void invoke () const = 0; + virtual ~ITestInvoker(); }; - struct IGeneratorsForTest { - virtual ~IGeneratorsForTest(); + class TestCase; + struct IConfig; - virtual IGeneratorInfo& getGeneratorInfo( std::string const& fileInfo, std::size_t size ) = 0; - virtual bool moveNext() = 0; + struct ITestCaseRegistry { + virtual ~ITestCaseRegistry(); + virtual std::vector const& getAllTests() const = 0; + virtual std::vector const& getAllTestsSorted( IConfig const& config ) const = 0; }; - IGeneratorsForTest* createGeneratorsForTest(); + bool isThrowSafe( TestCase const& testCase, IConfig const& config ); + bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ); + std::vector filterTests( std::vector const& testCases, TestSpec const& testSpec, IConfig const& config ); + std::vector const& getAllTestCasesSorted( IConfig const& config ); -} // end namespace Catch +} -// #included from: catch_ptr.hpp -#define TWOBLUECUBES_CATCH_PTR_HPP_INCLUDED +// end catch_interfaces_testcase.h +// start catch_stringref.h -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wpadded" -#endif +#include +#include +#include +#include namespace Catch { - // An intrusive reference counting smart pointer. - // T must implement addRef() and release() methods - // typically implementing the IShared interface - template - class Ptr { + /// A non-owning string class (similar to the forthcoming std::string_view) + /// Note that, because a StringRef may be a substring of another string, + /// it may not be null terminated. + class StringRef { public: - Ptr() : m_p( CATCH_NULL ){} - Ptr( T* p ) : m_p( p ){ - if( m_p ) - m_p->addRef(); - } - Ptr( Ptr const& other ) : m_p( other.m_p ){ - if( m_p ) - m_p->addRef(); - } - ~Ptr(){ - if( m_p ) - m_p->release(); - } - void reset() { - if( m_p ) - m_p->release(); - m_p = CATCH_NULL; - } - Ptr& operator = ( T* p ){ - Ptr temp( p ); - swap( temp ); - return *this; - } - Ptr& operator = ( Ptr const& other ){ - Ptr temp( other ); - swap( temp ); - return *this; - } - void swap( Ptr& other ) { std::swap( m_p, other.m_p ); } - T* get() const{ return m_p; } - T& operator*() const { return *m_p; } - T* operator->() const { return m_p; } - bool operator !() const { return m_p == CATCH_NULL; } - operator SafeBool::type() const { return SafeBool::makeSafe( m_p != CATCH_NULL ); } + using size_type = std::size_t; + using const_iterator = const char*; private: - T* m_p; - }; + static constexpr char const* const s_empty = ""; - struct IShared : NonCopyable { - virtual ~IShared(); - virtual void addRef() const = 0; - virtual void release() const = 0; - }; + char const* m_start = s_empty; + size_type m_size = 0; + + public: // construction + constexpr StringRef() noexcept = default; - template - struct SharedImpl : T { + StringRef( char const* rawChars ) noexcept; + + constexpr StringRef( char const* rawChars, size_type size ) noexcept + : m_start( rawChars ), + m_size( size ) + {} - SharedImpl() : m_rc( 0 ){} + StringRef( std::string const& stdString ) noexcept + : m_start( stdString.c_str() ), + m_size( stdString.size() ) + {} - virtual void addRef() const { - ++m_rc; + explicit operator std::string() const { + return std::string(m_start, m_size); } - virtual void release() const { - if( --m_rc == 0 ) - delete this; + + public: // operators + auto operator == ( StringRef const& other ) const noexcept -> bool; + auto operator != (StringRef const& other) const noexcept -> bool { + return !(*this == other); } - mutable unsigned int m_rc; - }; + auto operator[] ( size_type index ) const noexcept -> char { + assert(index < m_size); + return m_start[index]; + } -} // end namespace Catch + public: // named queries + constexpr auto empty() const noexcept -> bool { + return m_size == 0; + } + constexpr auto size() const noexcept -> size_type { + return m_size; + } -#ifdef __clang__ -#pragma clang diagnostic pop -#endif + // Returns the current start pointer. If the StringRef is not + // null-terminated, throws std::domain_exception + auto c_str() const -> char const*; -namespace Catch { + public: // substrings and searches + // Returns a substring of [start, start + length). + // If start + length > size(), then the substring is [start, size()). + // If start > size(), then the substring is empty. + auto substr( size_type start, size_type length ) const noexcept -> StringRef; - class TestCase; - class Stream; - struct IResultCapture; - struct IRunner; - struct IGeneratorsForTest; - struct IConfig; + // Returns the current start pointer. May not be null-terminated. + auto data() const noexcept -> char const*; - struct IContext - { - virtual ~IContext(); + constexpr auto isNullTerminated() const noexcept -> bool { + return m_start[m_size] == '\0'; + } - virtual IResultCapture* getResultCapture() = 0; - virtual IRunner* getRunner() = 0; - virtual size_t getGeneratorIndex( std::string const& fileInfo, size_t totalSize ) = 0; - virtual bool advanceGeneratorsForCurrentTest() = 0; - virtual Ptr getConfig() const = 0; + public: // iterators + constexpr const_iterator begin() const { return m_start; } + constexpr const_iterator end() const { return m_start + m_size; } }; - struct IMutableContext : IContext - { - virtual ~IMutableContext(); - virtual void setResultCapture( IResultCapture* resultCapture ) = 0; - virtual void setRunner( IRunner* runner ) = 0; - virtual void setConfig( Ptr const& config ) = 0; - }; + auto operator += ( std::string& lhs, StringRef const& sr ) -> std::string&; + auto operator << ( std::ostream& os, StringRef const& sr ) -> std::ostream&; - IContext& getCurrentContext(); - IMutableContext& getCurrentMutableContext(); - void cleanUpContext(); - Stream createStream( std::string const& streamName ); + constexpr auto operator "" _sr( char const* rawChars, std::size_t size ) noexcept -> StringRef { + return StringRef( rawChars, size ); + } +} // namespace Catch +constexpr auto operator "" _catch_sr( char const* rawChars, std::size_t size ) noexcept -> Catch::StringRef { + return Catch::StringRef( rawChars, size ); } -// #included from: internal/catch_test_registry.hpp -#define TWOBLUECUBES_CATCH_TEST_REGISTRY_HPP_INCLUDED +// end catch_stringref.h +// start catch_preprocessor.hpp -// #included from: catch_interfaces_testcase.h -#define TWOBLUECUBES_CATCH_INTERFACES_TESTCASE_H_INCLUDED -#include +#define CATCH_RECURSION_LEVEL0(...) __VA_ARGS__ +#define CATCH_RECURSION_LEVEL1(...) CATCH_RECURSION_LEVEL0(CATCH_RECURSION_LEVEL0(CATCH_RECURSION_LEVEL0(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL2(...) CATCH_RECURSION_LEVEL1(CATCH_RECURSION_LEVEL1(CATCH_RECURSION_LEVEL1(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL3(...) CATCH_RECURSION_LEVEL2(CATCH_RECURSION_LEVEL2(CATCH_RECURSION_LEVEL2(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL4(...) CATCH_RECURSION_LEVEL3(CATCH_RECURSION_LEVEL3(CATCH_RECURSION_LEVEL3(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL5(...) CATCH_RECURSION_LEVEL4(CATCH_RECURSION_LEVEL4(CATCH_RECURSION_LEVEL4(__VA_ARGS__))) -namespace Catch { +#ifdef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +#define INTERNAL_CATCH_EXPAND_VARGS(...) __VA_ARGS__ +// MSVC needs more evaluations +#define CATCH_RECURSION_LEVEL6(...) CATCH_RECURSION_LEVEL5(CATCH_RECURSION_LEVEL5(CATCH_RECURSION_LEVEL5(__VA_ARGS__))) +#define CATCH_RECURSE(...) CATCH_RECURSION_LEVEL6(CATCH_RECURSION_LEVEL6(__VA_ARGS__)) +#else +#define CATCH_RECURSE(...) CATCH_RECURSION_LEVEL5(__VA_ARGS__) +#endif - class TestSpec; +#define CATCH_REC_END(...) +#define CATCH_REC_OUT + +#define CATCH_EMPTY() +#define CATCH_DEFER(id) id CATCH_EMPTY() + +#define CATCH_REC_GET_END2() 0, CATCH_REC_END +#define CATCH_REC_GET_END1(...) CATCH_REC_GET_END2 +#define CATCH_REC_GET_END(...) CATCH_REC_GET_END1 +#define CATCH_REC_NEXT0(test, next, ...) next CATCH_REC_OUT +#define CATCH_REC_NEXT1(test, next) CATCH_DEFER ( CATCH_REC_NEXT0 ) ( test, next, 0) +#define CATCH_REC_NEXT(test, next) CATCH_REC_NEXT1(CATCH_REC_GET_END test, next) + +#define CATCH_REC_LIST0(f, x, peek, ...) , f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1) ) ( f, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST1(f, x, peek, ...) , f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST0) ) ( f, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST2(f, x, peek, ...) f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1) ) ( f, peek, __VA_ARGS__ ) + +#define CATCH_REC_LIST0_UD(f, userdata, x, peek, ...) , f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1_UD) ) ( f, userdata, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST1_UD(f, userdata, x, peek, ...) , f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST0_UD) ) ( f, userdata, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST2_UD(f, userdata, x, peek, ...) f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1_UD) ) ( f, userdata, peek, __VA_ARGS__ ) + +// Applies the function macro `f` to each of the remaining parameters, inserts commas between the results, +// and passes userdata as the first parameter to each invocation, +// e.g. CATCH_REC_LIST_UD(f, x, a, b, c) evaluates to f(x, a), f(x, b), f(x, c) +#define CATCH_REC_LIST_UD(f, userdata, ...) CATCH_RECURSE(CATCH_REC_LIST2_UD(f, userdata, __VA_ARGS__, ()()(), ()()(), ()()(), 0)) + +#define CATCH_REC_LIST(f, ...) CATCH_RECURSE(CATCH_REC_LIST2(f, __VA_ARGS__, ()()(), ()()(), ()()(), 0)) + +#define INTERNAL_CATCH_EXPAND1(param) INTERNAL_CATCH_EXPAND2(param) +#define INTERNAL_CATCH_EXPAND2(...) INTERNAL_CATCH_NO## __VA_ARGS__ +#define INTERNAL_CATCH_DEF(...) INTERNAL_CATCH_DEF __VA_ARGS__ +#define INTERNAL_CATCH_NOINTERNAL_CATCH_DEF +#define INTERNAL_CATCH_STRINGIZE(...) INTERNAL_CATCH_STRINGIZE2(__VA_ARGS__) +#ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +#define INTERNAL_CATCH_STRINGIZE2(...) #__VA_ARGS__ +#define INTERNAL_CATCH_STRINGIZE_WITHOUT_PARENS(param) INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_REMOVE_PARENS(param)) +#else +// MSVC is adding extra space and needs another indirection to expand INTERNAL_CATCH_NOINTERNAL_CATCH_DEF +#define INTERNAL_CATCH_STRINGIZE2(...) INTERNAL_CATCH_STRINGIZE3(__VA_ARGS__) +#define INTERNAL_CATCH_STRINGIZE3(...) #__VA_ARGS__ +#define INTERNAL_CATCH_STRINGIZE_WITHOUT_PARENS(param) (INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_REMOVE_PARENS(param)) + 1) +#endif - struct ITestCase : IShared { - virtual void invoke () const = 0; - protected: - virtual ~ITestCase(); - }; +#define INTERNAL_CATCH_MAKE_NAMESPACE2(...) ns_##__VA_ARGS__ +#define INTERNAL_CATCH_MAKE_NAMESPACE(name) INTERNAL_CATCH_MAKE_NAMESPACE2(name) - class TestCase; - struct IConfig; +#define INTERNAL_CATCH_REMOVE_PARENS(...) INTERNAL_CATCH_EXPAND1(INTERNAL_CATCH_DEF __VA_ARGS__) - struct ITestCaseRegistry { - virtual ~ITestCaseRegistry(); - virtual std::vector const& getAllTests() const = 0; - virtual std::vector const& getAllTestsSorted( IConfig const& config ) const = 0; +#ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +#define INTERNAL_CATCH_MAKE_TYPE_LIST2(...) decltype(get_wrapper()) +#define INTERNAL_CATCH_MAKE_TYPE_LIST(...) INTERNAL_CATCH_MAKE_TYPE_LIST2(INTERNAL_CATCH_REMOVE_PARENS(__VA_ARGS__)) +#else +#define INTERNAL_CATCH_MAKE_TYPE_LIST2(...) INTERNAL_CATCH_EXPAND_VARGS(decltype(get_wrapper())) +#define INTERNAL_CATCH_MAKE_TYPE_LIST(...) INTERNAL_CATCH_EXPAND_VARGS(INTERNAL_CATCH_MAKE_TYPE_LIST2(INTERNAL_CATCH_REMOVE_PARENS(__VA_ARGS__))) +#endif + +#define INTERNAL_CATCH_MAKE_TYPE_LISTS_FROM_TYPES(...)\ + CATCH_REC_LIST(INTERNAL_CATCH_MAKE_TYPE_LIST,__VA_ARGS__) + +#define INTERNAL_CATCH_REMOVE_PARENS_1_ARG(_0) INTERNAL_CATCH_REMOVE_PARENS(_0) +#define INTERNAL_CATCH_REMOVE_PARENS_2_ARG(_0, _1) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_1_ARG(_1) +#define INTERNAL_CATCH_REMOVE_PARENS_3_ARG(_0, _1, _2) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_2_ARG(_1, _2) +#define INTERNAL_CATCH_REMOVE_PARENS_4_ARG(_0, _1, _2, _3) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_3_ARG(_1, _2, _3) +#define INTERNAL_CATCH_REMOVE_PARENS_5_ARG(_0, _1, _2, _3, _4) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_4_ARG(_1, _2, _3, _4) +#define INTERNAL_CATCH_REMOVE_PARENS_6_ARG(_0, _1, _2, _3, _4, _5) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_5_ARG(_1, _2, _3, _4, _5) +#define INTERNAL_CATCH_REMOVE_PARENS_7_ARG(_0, _1, _2, _3, _4, _5, _6) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_6_ARG(_1, _2, _3, _4, _5, _6) +#define INTERNAL_CATCH_REMOVE_PARENS_8_ARG(_0, _1, _2, _3, _4, _5, _6, _7) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_7_ARG(_1, _2, _3, _4, _5, _6, _7) +#define INTERNAL_CATCH_REMOVE_PARENS_9_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_8_ARG(_1, _2, _3, _4, _5, _6, _7, _8) +#define INTERNAL_CATCH_REMOVE_PARENS_10_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_9_ARG(_1, _2, _3, _4, _5, _6, _7, _8, _9) +#define INTERNAL_CATCH_REMOVE_PARENS_11_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_10_ARG(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10) + +#define INTERNAL_CATCH_VA_NARGS_IMPL(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, N, ...) N + +#define INTERNAL_CATCH_TYPE_GEN\ + template struct TypeList {};\ + template\ + constexpr auto get_wrapper() noexcept -> TypeList { return {}; }\ + template class...> struct TemplateTypeList{};\ + template class...Cs>\ + constexpr auto get_wrapper() noexcept -> TemplateTypeList { return {}; }\ + template\ + struct append;\ + template\ + struct rewrap;\ + template class, typename...>\ + struct create;\ + template class, typename>\ + struct convert;\ + \ + template \ + struct append { using type = T; };\ + template< template class L1, typename...E1, template class L2, typename...E2, typename...Rest>\ + struct append, L2, Rest...> { using type = typename append, Rest...>::type; };\ + template< template class L1, typename...E1, typename...Rest>\ + struct append, TypeList, Rest...> { using type = L1; };\ + \ + template< template class Container, template class List, typename...elems>\ + struct rewrap, List> { using type = TypeList>; };\ + template< template class Container, template class List, class...Elems, typename...Elements>\ + struct rewrap, List, Elements...> { using type = typename append>, typename rewrap, Elements...>::type>::type; };\ + \ + template