CUMULUS-3135 - Update integration test scripts to fail on test timeout (

#3401)

* Update integration test scripts to fail on test timeout

* Fixup

* Fixup

* Update script interpreter for test runs

* Fix script

* Fixup

* Fixup

* Update timeout pass/fail conditional

CHANGELOG.md

@@ -18,6 +18,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - **CUMULUS-3115**
   - Fixed DiscoverGranules' workflow's duplicateHandling when set to `skip` or `error` to stop retrying
     after receiving a 404 Not Found Response Error from the `cumulus-api`.
+- **CUMULUS-3315**
+  - Update CI scripts to use shell logic/GNU timeout to bound test timeouts
+    instead of NPM `parallel` package, as timeouts were not resulting in
+    integration test failure
 - **CUMULUS-3223**
   - Update `@cumulus/cmrjs/cmr-utils.getGranuleTemporalInfo` to handle the error when the cmr file s3url is not available
   - Update `sfEventSqsToDbRecords` lambda to return [partial batch failure](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting),

example/scripts/run_test.sh

@@ -2,22 +2,26 @@
 
 set +e
 
-specName=$(echo "$2" | rev | cut -d'/' -f 1 | cut -d'.' -f 2 | rev)
+specName=$(echo "$3" | rev | cut -d'/' -f 1 | cut -d'.' -f 2 | rev)
 outputPath="${1}/${specName}-running.txt"
+testTimeout=$2
 
 TIMESTAMP=$(date "+%Y-%m-%dT%H:%M:%S")
-echo "$TIMESTAMP ../node_modules/.bin/jasmine $2 STARTED"
-
-../node_modules/.bin/jasmine --no-color "$2" > "$outputPath" 2>&1
+echo "$TIMESTAMP ../node_modules/.bin/jasmine $3 STARTED"
+timeout "$testTimeout" ../node_modules/.bin/jasmine --no-color "$3" > "$outputPath" 2>&1
 result=$?
 
 TIMESTAMP=$(date "+%Y-%m-%dT%H:%M:%S")
 if [ "$result" -eq "0" ]; then
-  echo "$TIMESTAMP ../node_modules/.bin/jasmine $2 PASSED"
+  echo "$TIMESTAMP ../node_modules/.bin/jasmine $3 PASSED"
   mv "$outputPath" "$1/${specName}-passed.txt"
+elif { [ "$result" -gt "124" ] && [ "$result" -lt "128" ] ;} || [ "$result" -eq "137" ]; then
+  echo "$TIMESTAMP ../node_modules/.bin/jasmine $3 FAILED -- TIMEOUT"
+  mv "$outputPath" "$1/${specName}-failed.txt"
+  result=1
 else
-  echo "$TIMESTAMP ../node_modules/.bin/jasmine $2 FAILED"
+  echo "$TIMESTAMP ../node_modules/.bin/jasmine $3 FAILED"
   mv "$outputPath" "$1/${specName}-failed.txt"
 fi
 
-exit $result
+exit $result

example/scripts/tests-parallel.sh

@@ -10,11 +10,12 @@ DOT_PID="$!"
 
 TESTS=$(find spec/parallel -type f -name '*spec.js' -or -name '*Spec.js')
 testOutputDir=scripts/test_output
+testTimeout=1200
 
 rm -r -f $testOutputDir
 mkdir -p $testOutputDir
 
-echo "" | ../node_modules/.bin/parallel -j "${INTEGRATION_CONCURRENCY:=0}" --timeout 1200 sh scripts/run_test.sh  $testOutputDir ::: $TESTS
+echo "" | ../node_modules/.bin/parallel -j "${INTEGRATION_CONCURRENCY:=0}" sh scripts/run_test.sh  $testOutputDir $testTimeout ::: $TESTS
 result=$?
 echo parallel tests complete: $result suite failures