chore: convert expectation files to policies and use in integration t…

…ests (#760)

Signed-off-by: Ben Selwyn-Smith <benselwynsmith@googlemail.com>

scripts/dev_scripts/integration_tests_docker.sh

@@ -14,13 +14,19 @@ RUN_MACARON_SCRIPT=$2
 
 # The scripts to compare the results of the integration tests.
 COMPARE_DEPS=$WORKSPACE/tests/dependency_analyzer/compare_dependencies.py
-COMPARE_JSON_OUT=$WORKSPACE/tests/e2e/compare_e2e_result.py
 COMPARE_POLICIES=$WORKSPACE/tests/policy_engine/compare_policy_reports.py
 COMPARE_VSA=$WORKSPACE/tests/vsa/compare_vsa.py
 UNIT_TEST_SCRIPT=$WORKSPACE/scripts/dev_scripts/test_run_macaron_sh.py
+RUN_POLICY="$RUN_MACARON_SCRIPT verify-policy"
+DB=$WORKSPACE/output/macaron.db
 
 RESULT_CODE=0
 
+function run_macaron_clean() {
+    rm $DB
+    $RUN_MACARON_SCRIPT "$@"
+}
+
 function log_fail() {
     printf "Error: FAILED integration test (line ${BASH_LINENO}) %s\n" $@
     RESULT_CODE=1
@@ -37,49 +43,47 @@ echo "with dependency resolution using cyclonedx Gradle plugin (default)."
 echo -e "----------------------------------------------------------------------------------\n"
 DEP_RESULT=$WORKSPACE/output/reports/github_com/timyarkov/multibuild_test/dependencies.json
 DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test.json
-JSON_RESULT=$WORKSPACE/output/reports/github_com/timyarkov/multibuild_test/multibuild_test.json
-JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/multibuild_test/multibuild_test.json
-$RUN_MACARON_SCRIPT analyze -rp https://github.com/timyarkov/multibuild_test -b main -d a8b0efe24298bc81f63217aaa84776c3d48976c5 || log_fail
+OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/multibuild_test/multibuild_test.dl
+run_macaron_clean analyze -rp https://github.com/timyarkov/multibuild_test -b main -d a8b0efe24298bc81f63217aaa84776c3d48976c5 || log_fail
 
 python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail
 
-python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
+$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail
 
 echo -e "\n----------------------------------------------------------------------------------"
 echo "apache/maven: Check the resolved dependency output with config for cyclonedx maven plugin (default)."
 echo -e "----------------------------------------------------------------------------------\n"
 DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json
 DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_apache_maven.json
 
-$RUN_MACARON_SCRIPT analyze -c $WORKSPACE/tests/dependency_analyzer/configurations/maven_config.yaml || log_fail
+run_macaron_clean analyze -c $WORKSPACE/tests/dependency_analyzer/configurations/maven_config.yaml || log_fail
 python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail
 
 echo -e "\n----------------------------------------------------------------------------------"
 echo "apache/maven: e2e using the local repo path, the branch name and the commit digest without dependency resolution."
 echo -e "----------------------------------------------------------------------------------\n"
-JSON_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/maven.json
-JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/maven/maven.json
+OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/maven/maven.dl
+
+run_macaron_clean -lr $WORKSPACE/output/git_repos/github_com analyze -r apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail
 
-$RUN_MACARON_SCRIPT -lr $WORKSPACE/output/git_repos/github_com analyze -r apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail
-python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
+$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail
 
 echo -e "\n----------------------------------------------------------------------------------"
 echo "apache/maven: Check the e2e output JSON file with config and no dependency analyzing."
 echo -e "----------------------------------------------------------------------------------\n"
-JSON_RESULT_DIR=$WORKSPACE/output/reports/github_com/apache/maven
-JSON_EXPECT_DIR=$WORKSPACE/tests/e2e/expected_results/maven
+EXPECT_DIR=$WORKSPACE/tests/e2e/expected_results/maven
 
 declare -a COMPARE_FILES=(
-    "maven.json"
-    "guava.json"
-    "mockito.json"
+    "maven.dl"
+    "guava.dl"
+    "mockito.dl"
 )
 
-$RUN_MACARON_SCRIPT analyze -c $WORKSPACE/tests/e2e/configurations/maven_config.yaml --skip-deps || log_fail
+run_macaron_clean analyze -c $WORKSPACE/tests/e2e/configurations/maven_config.yaml --skip-deps || log_fail
 
 for i in "${COMPARE_FILES[@]}"
 do
-    python $COMPARE_JSON_OUT $JSON_RESULT_DIR/$i $JSON_EXPECT_DIR/$i || log_fail
+    $RUN_POLICY -d $DB -f $EXPECT_DIR/$i || log_fail
 done
 
 echo -e "\n----------------------------------------------------------------------------------"
@@ -89,63 +93,47 @@ SBOM_FILE=$WORKSPACE/tests/dependency_analyzer/cyclonedx/resources/apache_maven_
 DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/apache_maven_with_sbom_provided.json
 DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json
 
-$RUN_MACARON_SCRIPT analyze -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b -sbom $SBOM_FILE || log_fail
+run_macaron_clean analyze -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b -sbom $SBOM_FILE || log_fail
 
 python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail
 
 echo -e "\n----------------------------------------------------------------------------------"
 echo "apache/maven: Analyzing with PURL and repository path without dependency resolution."
 echo -e "----------------------------------------------------------------------------------\n"
-JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/purl/maven/maven.json
-JSON_RESULT=$WORKSPACE/output/reports/maven/apache/maven/maven.json
-$RUN_MACARON_SCRIPT analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail
+JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/purl/maven/maven.dl
+run_macaron_clean analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail
 
-python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
+$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail
 
 echo -e "\n----------------------------------------------------------------------------------"
 echo "urllib3/urllib3: Analyzing the repo path when automatic dependency resolution is skipped."
 echo "The CUE expectation file is provided as a single file path."
 echo -e "----------------------------------------------------------------------------------\n"
-JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.json
-JSON_RESULT=$WORKSPACE/output/reports/github_com/urllib3/urllib3/urllib3.json
+JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.dl
 EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/urllib3_PASS.cue
-$RUN_MACARON_SCRIPT analyze -pe $EXPECTATION_FILE -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail
+run_macaron_clean analyze -pe $EXPECTATION_FILE -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail
 
-python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
+$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail
 
 echo -e "\n----------------------------------------------------------------------------------"
 echo "urllib3/urllib3: Analyzing the repo path when automatic dependency resolution is skipped."
 echo "The CUE expectation file should be found via the directory path."
 echo -e "----------------------------------------------------------------------------------\n"
-JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.json
-JSON_RESULT=$WORKSPACE/output/reports/github_com/urllib3/urllib3/urllib3.json
+JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.dl
 EXPECTATION_DIR=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/
-$RUN_MACARON_SCRIPT analyze -pe $EXPECTATION_DIR -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail
+run_macaron_clean analyze -pe $EXPECTATION_DIR -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail
 
-python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
+$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail
 
 echo -e "\n----------------------------------------------------------------------------------"
 echo "Test verifying CUE provenance expectation for ossf/scorecard"
 echo -e "----------------------------------------------------------------------------------\n"
-JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/scorecard/scorecard.json
-JSON_RESULT=$WORKSPACE/output/reports/github/ossf/scorecard/scorecard.json
+JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/scorecard/scorecard.dl
 DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/scorecard.ini
 EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/scorecard_PASS.cue
-$RUN_MACARON_SCRIPT -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -purl pkg:github/ossf/scorecard@v4.13.1 --skip-deps || log_fail
+run_macaron_clean -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -purl pkg:github/ossf/scorecard@v4.13.1 --skip-deps || log_fail
 
-python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
-
-echo -e "\n----------------------------------------------------------------------------------"
-echo "slsa-framework/slsa-verifier: Analyzing the repo path when automatic dependency resolution is skipped"
-echo "and CUE file is provided as expectation."
-echo -e "----------------------------------------------------------------------------------\n"
-JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.json
-JSON_RESULT=$WORKSPACE/output/reports/github_com/slsa-framework/slsa-verifier/slsa-verifier.json
-EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/slsa_verifier_PASS.cue
-DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/slsa_verifier.ini
-$RUN_MACARON_SCRIPT -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -rp https://github.com/slsa-framework/slsa-verifier -b main -d fc50b662fcfeeeb0e97243554b47d9b20b14efac --skip-deps || log_fail
-
-python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
+$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail
 
 echo -e "\n----------------------------------------------------------------------------------"
 echo "Run policy CLI with scorecard results."
@@ -160,11 +148,21 @@ $RUN_MACARON_SCRIPT verify-policy -f $POLICY_FILE -d "$WORKSPACE/output/macaron.
 python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail
 python "$COMPARE_VSA" "$VSA_RESULT" "$VSA_PAYLOAD_EXPECTED" || log_fail
 
+echo -e "\n----------------------------------------------------------------------------------"
+echo "slsa-framework/slsa-verifier: Analyzing the repo path when automatic dependency resolution is skipped"
+echo "and CUE file is provided as expectation."
+echo -e "----------------------------------------------------------------------------------\n"
+JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.dl
+EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/slsa_verifier_PASS.cue
+DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/slsa_verifier.ini
+run_macaron_clean -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -rp https://github.com/slsa-framework/slsa-verifier -b main -d fc50b662fcfeeeb0e97243554b47d9b20b14efac --skip-deps || log_fail
+
+$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail
+
 echo -e "\n----------------------------------------------------------------------------------"
 echo "behnazh-w/example-maven-app as a local and remote repository"
 echo "Test the Witness and GitHub provenances as an input, Cue expectation validation, Policy CLI and VSA generation."
 echo -e "----------------------------------------------------------------------------------\n"
-RUN_POLICY="macaron verify-policy"
 POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/example-maven-project/policy.dl
 POLICY_RESULT=$WORKSPACE/output/policy_report.json
 POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/example-maven-project/example_maven_project_policy_report.json
@@ -179,7 +177,7 @@ WITNESS_PROVENANCE_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/resources/vali
 git clone https://github.com/behnazh-w/example-maven-app.git $WORKSPACE/output/git_repos/local_repos/example-maven-app || log_fail
 
 # Check the Witness provenance.
-$RUN_MACARON_SCRIPT analyze -pf $WITNESS_PROVENANCE_FILE -pe $WITNESS_EXPECTATION_FILE -purl pkg:maven/io.github.behnazh-w.demo/example-maven-app@1.0-SNAPSHOT?type=jar --repo-path example-maven-app --skip-deps || log_fail
+run_macaron_clean analyze -pf $WITNESS_PROVENANCE_FILE -pe $WITNESS_EXPECTATION_FILE -purl pkg:maven/io.github.behnazh-w.demo/example-maven-app@1.0-SNAPSHOT?type=jar --repo-path example-maven-app --skip-deps || log_fail
 
 # Test the remote repo with GitHub provenance.
 GITHUB_EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/github-example-maven-project.cue

src/macaron/policy_engine/prelude/helper_rules.dl

@@ -1,4 +1,4 @@
-/* Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. */
+/* Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved. */
 /* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */
 
 /* Souffle datalog rules to assist in policy authoring */
@@ -9,6 +9,9 @@
 .decl check_passed(component_id: number, check_name: symbol)
 check_passed(component_id, check_name) :- check_result(_, check_name, 1, component_id).
 
+.decl check_failed(component_id: number, check_name: symbol)
+check_failed(component_id, check_name) :- check_result(_, check_name, 0, component_id).
+
 /**
  * Check name is valid.
  */
@@ -42,6 +45,12 @@ transitive_dependency(parent, dependency) :-
 .decl is_repo(repo_id: number, repo_complete_name: symbol, component_id: number)
  is_repo(repo_id, repo_complete_name, component_id) :- repository(repo_id, component_id, repo_complete_name, _, _, _, _, _, _, _, _, _, _).
 
+/**
+ *  Extract the id and URL from the repository relation.
+ */
+.decl is_repo_url(repo_id: number, url: symbol)
+ is_repo_url(repo_id, url) :- repository(_, repo_id, _, _, _, _, _, url, _, _, _, _, _).
+
 
 /**
  * A convenience relation to find analyses conducted on a specific repository.

tests/e2e/expected_results/docker_test/docker_test.dl

@@ -0,0 +1,22 @@
+/* Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. */
+/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */
+
+#include "prelude.dl"
+
+Policy("test_policy", component_id, "") :-
+    check_passed(component_id, "mcn_build_as_code_1"),
+    check_passed(component_id, "mcn_build_script_1"),
+    check_passed(component_id, "mcn_build_service_1"),
+    check_passed(component_id, "mcn_version_control_system_1"),
+    check_failed(component_id, "mcn_infer_artifact_pipeline_1"),
+    check_failed(component_id, "mcn_provenance_available_1"),
+    check_failed(component_id, "mcn_provenance_derived_commit_1"),
+    check_failed(component_id, "mcn_provenance_derived_repo_1"),
+    check_failed(component_id, "mcn_provenance_expectation_1"),
+    check_failed(component_id, "mcn_provenance_level_three_1"),
+    check_failed(component_id, "mcn_provenance_witness_level_one_1"),
+    check_failed(component_id, "mcn_trusted_builder_level_three_1"),
+    is_repo_url(component_id, "https://github.com/timyarkov/docker_test").
+
+apply_policy_to("test_policy", component_id) :-
+    is_component(component_id, "pkg:github.com/timyarkov/docker_test@404a51a2f38c4470af6b32e4e00b5318c2d7c0cc").