From 7b5b394a55adf0d4125d84ff153a8f1658aa7fe5 Mon Sep 17 00:00:00 2001 From: behnazh-w Date: Fri, 5 Jan 2024 11:50:18 +1000 Subject: [PATCH 1/2] test: fix the GitHub Actions workflow detection test Signed-off-by: behnazh-w --- .../ci_service/test_github_actions.py | 166 ++++++++++-------- 1 file changed, 90 insertions(+), 76 deletions(-) diff --git a/tests/slsa_analyzer/ci_service/test_github_actions.py b/tests/slsa_analyzer/ci_service/test_github_actions.py index 00830f76b..b02d218d6 100644 --- a/tests/slsa_analyzer/ci_service/test_github_actions.py +++ b/tests/slsa_analyzer/ci_service/test_github_actions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module tests GitHub Actions CI service.""" @@ -6,82 +6,96 @@ import os from pathlib import Path +import pytest + +from macaron import MACARON_PATH from macaron.code_analyzer.call_graph import CallGraph from macaron.parsers.actionparser import parse as parse_action from macaron.slsa_analyzer.ci_service.github_actions import GHWorkflowType, GitHubActions, GitHubNode -from ...macaron_testcase import MacaronTestCase - - -class TestGitHubActions(MacaronTestCase): - """Test the GitHub Actions CI service.""" - - github_actions = GitHubActions() - mock_repos = Path(__file__).parent.joinpath("mock_repos") - ga_has_build_kws = mock_repos.joinpath("has_build_gh_actions") - jenkins_build = mock_repos.joinpath("has_build_jenkins") - ga_no_build_kws = mock_repos.joinpath("no_build_gh_actions") - - def test_build_call_graph(self) -> None: - """Test building call graphs for GitHub Actions workflows.""" - resources_dir = Path(__file__).parent.joinpath("resources", "github") - - # Test internal and reusable workflows. - # Parse GitHub Actions workflows. - root = GitHubNode(name="root", node_type=GHWorkflowType.NONE, source_path="", parsed_obj={}, caller_path="") - gh_cg = CallGraph(root, "") - workflow_path = os.path.join(resources_dir, "valid1.yaml") - parsed_obj = parse_action(workflow_path, macaron_path=str(self.macaron_path)) - - callee = GitHubNode( - name=os.path.basename(workflow_path), - node_type=GHWorkflowType.INTERNAL, - source_path=workflow_path, - parsed_obj=parsed_obj, - caller_path="", - ) - root.add_callee(callee) - self.github_actions.build_call_graph_from_node(callee) - assert [ - "GitHubNode(valid1.yaml,GHWorkflowType.INTERNAL)", - "GitHubNode(apache/maven-gh-actions-shared/.github/workflows/maven-verify.yml@v2,GHWorkflowType.REUSABLE)", - ] == [str(node) for node in gh_cg.bfs()] - - # Test internal and external workflows. - # Parse GitHub Actions workflows. - root = GitHubNode(name="root", node_type=GHWorkflowType.NONE, source_path="", parsed_obj={}, caller_path="") - gh_cg = CallGraph(root, "") - workflow_path = os.path.join(resources_dir, "valid2.yaml") - parsed_obj = parse_action(workflow_path, macaron_path=str(self.macaron_path)) - - callee = GitHubNode( - name=os.path.basename(workflow_path), - node_type=GHWorkflowType.INTERNAL, - source_path=workflow_path, - parsed_obj=parsed_obj, - caller_path="", - ) - root.add_callee(callee) - self.github_actions.build_call_graph_from_node(callee) - assert [ - "GitHubNode(valid2.yaml,GHWorkflowType.INTERNAL)", - "GitHubNode(actions/checkout@v3,GHWorkflowType.EXTERNAL)", - "GitHubNode(actions/cache@v3,GHWorkflowType.EXTERNAL)", - "GitHubNode(actions/setup-java@v3,GHWorkflowType.EXTERNAL)", - ] == [str(node) for node in gh_cg.bfs()] - - def test_is_detected(self) -> None: - """Test detecting GitHub Action config files.""" - assert self.github_actions.is_detected(str(self.ga_has_build_kws)) - assert self.github_actions.is_detected(str(self.ga_no_build_kws)) - assert not self.github_actions.is_detected(str(self.jenkins_build)) - - def test_get_workflows(self) -> None: - """Test getting GitHub Actions workflows.""" - expect = list(self.ga_has_build_kws.joinpath(".github", "workflows").glob("*")).sort() - assert self.github_actions.get_workflows(str(self.ga_has_build_kws)).sort() == expect - - expect = list(self.ga_no_build_kws.joinpath(".github", "workflows").glob("*")).sort() - assert self.github_actions.get_workflows(str(self.ga_no_build_kws)).sort() == expect - - assert not self.github_actions.get_workflows(str(self.jenkins_build)) +mock_repos = Path(__file__).parent.joinpath("mock_repos") +ga_has_build_kws = mock_repos.joinpath("has_build_gh_actions") +jenkins_build = mock_repos.joinpath("has_build_jenkins") +ga_no_build_kws = mock_repos.joinpath("no_build_gh_actions") + + +@pytest.fixture(name="github_actions") +def github_actions_() -> GitHubActions: + """Create a GitHubActions instance.""" + return GitHubActions() + + +def test_build_call_graph(github_actions: GitHubActions) -> None: + """Test building call graphs for GitHub Actions workflows.""" + resources_dir = Path(__file__).parent.joinpath("resources", "github") + + # Test internal and reusable workflows. + # Parse GitHub Actions workflows. + root = GitHubNode(name="root", node_type=GHWorkflowType.NONE, source_path="", parsed_obj={}, caller_path="") + gh_cg = CallGraph(root, "") + workflow_path = os.path.join(resources_dir, "valid1.yaml") + parsed_obj = parse_action(workflow_path, macaron_path=MACARON_PATH) + + callee = GitHubNode( + name=os.path.basename(workflow_path), + node_type=GHWorkflowType.INTERNAL, + source_path=workflow_path, + parsed_obj=parsed_obj, + caller_path="", + ) + root.add_callee(callee) + github_actions.build_call_graph_from_node(callee) + assert [ + "GitHubNode(valid1.yaml,GHWorkflowType.INTERNAL)", + "GitHubNode(apache/maven-gh-actions-shared/.github/workflows/maven-verify.yml@v2,GHWorkflowType.REUSABLE)", + ] == [str(node) for node in gh_cg.bfs()] + + # Test internal and external workflows. + # Parse GitHub Actions workflows. + root = GitHubNode(name="root", node_type=GHWorkflowType.NONE, source_path="", parsed_obj={}, caller_path="") + gh_cg = CallGraph(root, "") + workflow_path = os.path.join(resources_dir, "valid2.yaml") + parsed_obj = parse_action(workflow_path, macaron_path=MACARON_PATH) + + callee = GitHubNode( + name=os.path.basename(workflow_path), + node_type=GHWorkflowType.INTERNAL, + source_path=workflow_path, + parsed_obj=parsed_obj, + caller_path="", + ) + root.add_callee(callee) + github_actions.build_call_graph_from_node(callee) + assert [ + "GitHubNode(valid2.yaml,GHWorkflowType.INTERNAL)", + "GitHubNode(actions/checkout@v3,GHWorkflowType.EXTERNAL)", + "GitHubNode(actions/cache@v3,GHWorkflowType.EXTERNAL)", + "GitHubNode(actions/setup-java@v3,GHWorkflowType.EXTERNAL)", + ] == [str(node) for node in gh_cg.bfs()] + + +def test_is_detected(github_actions: GitHubActions) -> None: + """Test detecting GitHub Action config files.""" + assert github_actions.is_detected(str(ga_has_build_kws)) + assert github_actions.is_detected(str(ga_no_build_kws)) + assert not github_actions.is_detected(str(jenkins_build)) + + +def test_get_workflows(github_actions: GitHubActions) -> None: + """Test detection of reachable GitHub Actions workflows.""" + # Test GitHub Actions workflows that contain build commands. + expect = [str(path) for path in ga_has_build_kws.joinpath(".github", "workflows").glob("*")] + expect.sort() + workflows = github_actions.get_workflows(str(ga_has_build_kws)) + workflows.sort() + assert workflows == expect + + # Test GitHub Actions workflows that do not contain build commands. + expect = [str(path) for path in ga_no_build_kws.joinpath(".github", "workflows").glob("*")] + expect.sort() + workflows = github_actions.get_workflows(str(ga_no_build_kws)) + workflows.sort() + assert workflows == expect + + # The GitHubActions workflow detection should not work on Jenkins CI configuration files. + assert not github_actions.get_workflows(str(jenkins_build)) From 9e4bfc9a903ed59566053e4df12e1db5d1755841 Mon Sep 17 00:00:00 2001 From: behnazh-w Date: Fri, 5 Jan 2024 14:59:14 +1000 Subject: [PATCH 2/2] chore: refactor the tests and use pytest's parametrize Signed-off-by: behnazh-w --- .../ci_service/test_github_actions.py | 96 ++++++++++--------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/tests/slsa_analyzer/ci_service/test_github_actions.py b/tests/slsa_analyzer/ci_service/test_github_actions.py index b02d218d6..afb972f9a 100644 --- a/tests/slsa_analyzer/ci_service/test_github_actions.py +++ b/tests/slsa_analyzer/ci_service/test_github_actions.py @@ -25,15 +25,42 @@ def github_actions_() -> GitHubActions: return GitHubActions() -def test_build_call_graph(github_actions: GitHubActions) -> None: +@pytest.mark.parametrize( + ( + "workflow_name", + "expect", + ), + [ + ( + "valid1.yaml", + [ + "GitHubNode(valid1.yaml,GHWorkflowType.INTERNAL)", + "GitHubNode(apache/maven-gh-actions-shared/.github/workflows/maven-verify.yml@v2,GHWorkflowType.REUSABLE)", + ], + ), + ( + "valid2.yaml", + [ + "GitHubNode(valid2.yaml,GHWorkflowType.INTERNAL)", + "GitHubNode(actions/checkout@v3,GHWorkflowType.EXTERNAL)", + "GitHubNode(actions/cache@v3,GHWorkflowType.EXTERNAL)", + "GitHubNode(actions/setup-java@v3,GHWorkflowType.EXTERNAL)", + ], + ), + ], + ids=[ + "Internal and reusable workflows", + "Internal and external workflows", + ], +) +def test_build_call_graph(github_actions: GitHubActions, workflow_name: str, expect: list[str]) -> None: """Test building call graphs for GitHub Actions workflows.""" resources_dir = Path(__file__).parent.joinpath("resources", "github") - # Test internal and reusable workflows. # Parse GitHub Actions workflows. root = GitHubNode(name="root", node_type=GHWorkflowType.NONE, source_path="", parsed_obj={}, caller_path="") gh_cg = CallGraph(root, "") - workflow_path = os.path.join(resources_dir, "valid1.yaml") + workflow_path = os.path.join(resources_dir, workflow_name) parsed_obj = parse_action(workflow_path, macaron_path=MACARON_PATH) callee = GitHubNode( @@ -45,33 +72,7 @@ def test_build_call_graph(github_actions: GitHubActions) -> None: ) root.add_callee(callee) github_actions.build_call_graph_from_node(callee) - assert [ - "GitHubNode(valid1.yaml,GHWorkflowType.INTERNAL)", - "GitHubNode(apache/maven-gh-actions-shared/.github/workflows/maven-verify.yml@v2,GHWorkflowType.REUSABLE)", - ] == [str(node) for node in gh_cg.bfs()] - - # Test internal and external workflows. - # Parse GitHub Actions workflows. - root = GitHubNode(name="root", node_type=GHWorkflowType.NONE, source_path="", parsed_obj={}, caller_path="") - gh_cg = CallGraph(root, "") - workflow_path = os.path.join(resources_dir, "valid2.yaml") - parsed_obj = parse_action(workflow_path, macaron_path=MACARON_PATH) - - callee = GitHubNode( - name=os.path.basename(workflow_path), - node_type=GHWorkflowType.INTERNAL, - source_path=workflow_path, - parsed_obj=parsed_obj, - caller_path="", - ) - root.add_callee(callee) - github_actions.build_call_graph_from_node(callee) - assert [ - "GitHubNode(valid2.yaml,GHWorkflowType.INTERNAL)", - "GitHubNode(actions/checkout@v3,GHWorkflowType.EXTERNAL)", - "GitHubNode(actions/cache@v3,GHWorkflowType.EXTERNAL)", - "GitHubNode(actions/setup-java@v3,GHWorkflowType.EXTERNAL)", - ] == [str(node) for node in gh_cg.bfs()] + assert [str(node) for node in gh_cg.bfs()] == expect def test_is_detected(github_actions: GitHubActions) -> None: @@ -81,21 +82,24 @@ def test_is_detected(github_actions: GitHubActions) -> None: assert not github_actions.is_detected(str(jenkins_build)) -def test_get_workflows(github_actions: GitHubActions) -> None: +@pytest.mark.parametrize( + "mock_repo", + [ + ga_has_build_kws, + ga_no_build_kws, + ], + ids=[ + "GH Actions with build", + "GH Actions with no build", + ], +) +def test_gh_get_workflows(github_actions: GitHubActions, mock_repo: Path) -> None: """Test detection of reachable GitHub Actions workflows.""" - # Test GitHub Actions workflows that contain build commands. - expect = [str(path) for path in ga_has_build_kws.joinpath(".github", "workflows").glob("*")] - expect.sort() - workflows = github_actions.get_workflows(str(ga_has_build_kws)) - workflows.sort() - assert workflows == expect - - # Test GitHub Actions workflows that do not contain build commands. - expect = [str(path) for path in ga_no_build_kws.joinpath(".github", "workflows").glob("*")] - expect.sort() - workflows = github_actions.get_workflows(str(ga_no_build_kws)) - workflows.sort() - assert workflows == expect - - # The GitHubActions workflow detection should not work on Jenkins CI configuration files. + expect = [str(path) for path in mock_repo.joinpath(".github", "workflows").glob("*")] + workflows = github_actions.get_workflows(str(mock_repo)) + assert sorted(workflows) == sorted(expect) + + +def test_gh_get_workflows_fail_on_jenkins(github_actions: GitHubActions) -> None: + """Assert GitHubActions workflow detection not working on Jenkins CI configuration files.""" assert not github_actions.get_workflows(str(jenkins_build))