chore: improve token import (#605)

Signed-off-by: Ben Selwyn-Smith <benselwynsmith@googlemail.com>

scripts/release_scripts/run_macaron.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 # This script runs the Macaron Docker image.
@@ -48,6 +48,9 @@ if [[ -z ${MACARON_IMAGE_TAG:-} ]]; then
     MACARON_IMAGE_TAG="latest"
 fi
 
+# This file is used to store token environment variables that will later be read by Macaron.
+TOKEN_FILE=".macaron_env_file"
+
 IMAGE="ghcr.io/oracle/macaron"
 
 # Workspace directory inside of the container.
@@ -255,6 +258,17 @@ function mount_file() {
     mounts+=("-v" "${file_on_host}:${file_in_container}:${mount_option}")
 }
 
+# Handle tokens.
+set +u
+echo "" > ${TOKEN_FILE}
+{
+    echo "GITHUB_TOKEN=${GITHUB_TOKEN}" >> ${TOKEN_FILE}
+    echo "MCN_GITLAB_TOKEN=${MCN_GITLAB_TOKEN}" >> ${TOKEN_FILE}
+    echo "MCN_SELF_HOSTED_GITLAB_TOKEN=${MCN_SELF_HOSTED_GITLAB_TOKEN}"
+} >> ${TOKEN_FILE}
+mount_file "macaron_env_file" ${TOKEN_FILE} ${MACARON_WORKSPACE}/${TOKEN_FILE} "rw,Z"
+set -u
+
 # Parse main arguments.
 while [[ $# -gt 0 ]]; do
     case $1 in
@@ -541,12 +555,11 @@ docker run \
     --rm -i "${tty[@]}" \
     -e "USER_UID=${USER_UID}" \
     -e "USER_GID=${USER_GID}" \
-    -e GITHUB_TOKEN \
-    -e MCN_GITLAB_TOKEN \
-    -e MCN_SELF_HOSTED_GITLAB_TOKEN \
     "${proxy_vars[@]}" \
     "${prod_vars[@]}" \
     "${mounts[@]}" \
     "${IMAGE}:${MACARON_IMAGE_TAG}" \
     "${entrypoint[@]}" \
     "${macaron_args[@]}"
+
+rm -f "$TOKEN_FILE"

src/macaron/__main__.py

@@ -187,13 +187,9 @@ def perform_action(action_args: argparse.Namespace) -> None:
             sys.exit(verify_policy(action_args))
 
         case "analyze":
-            # Check that the GitHub token is enabled.
-            gh_token = os.environ.get("GITHUB_TOKEN")
-            if not gh_token:
+            if not global_config.gh_token:
                 logger.error("GitHub access token not set.")
                 sys.exit(os.EX_USAGE)
-            global_config.gh_token = gh_token
-
             # TODO: Here we should try to statically analyze the config before
             # actually running the analysis.
             try:
@@ -221,6 +217,36 @@ def main(argv: list[str] | None = None) -> None:
         If ``argv`` is ``None``, argparse automatically looks at ``sys.argv``.
         Hence, we set ``argv = None`` by default.
     """
+    # Handle presence of token file. When running Macaron as a container, this file is created by the "run_macaron.sh"
+    # script and populated there. If Macaron is being run outside of a container, the token file should not exist, and
+    # tokens should be read directly from the environment instead.
+    token_file = os.path.join(os.getcwd(), ".macaron_env_file")
+    token_dict = {}
+    if os.path.exists(token_file):
+        # Read values into dictionary.
+        try:
+            with open(token_file, encoding="utf-8") as file:
+                for line in file:
+                    if not line or "=" not in line:
+                        continue
+                    key, value = line.rstrip().split("=", 1)
+                    token_dict[key] = value
+        except OSError as error:
+            logger.error("Could not open token file %s: %s", token_file, error)
+            sys.exit(os.EX_OSFILE)
+        # Overwrite file contents.
+        try:
+            with open(token_file, "w", encoding="utf-8"):
+                pass
+        except OSError as error:
+            logger.error("Could not overwrite token file %s: %s", token_file, error)
+            sys.exit(os.EX_OSFILE)
+
+    # Check presence of tokens in dictionary or environment, preferring the former.
+    global_config.gh_token = _get_token_from_dict_or_env("GITHUB_TOKEN", token_dict)
+    global_config.gl_token = _get_token_from_dict_or_env("MCN_GITLAB_TOKEN", token_dict)
+    global_config.gl_self_host_token = _get_token_from_dict_or_env("MCN_SELF_HOSTED_GITLAB_TOKEN", token_dict)
+
     main_parser = argparse.ArgumentParser(prog="macaron")
 
     main_parser.add_argument(
@@ -435,5 +461,10 @@ def main(argv: list[str] | None = None) -> None:
     perform_action(args)
 
 
+def _get_token_from_dict_or_env(token: str, token_dict: dict[str, str]) -> str:
+    """Return the value of passed token from passed dictionary or os environment."""
+    return token_dict[token] if token in token_dict else os.environ.get(token) or ""
+
+
 if __name__ == "__main__":
     main()

src/macaron/config/global_config.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 """This module contains the GlobalConfig class to be used globally."""
@@ -19,6 +19,8 @@ class GlobalConfig:
     build_log_path: str = ""
     local_repos_path: str = ""
     gh_token: str = ""
+    gl_token: str = ""
+    gl_self_host_token: str = ""
     debug_level: int = logging.DEBUG
     resources_path: str = ""
 

src/macaron/slsa_analyzer/git_service/gitlab.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 """This module contains the spec for the GitLab service.
@@ -19,13 +19,14 @@
 """
 
 import logging
-import os
 from abc import abstractmethod
+from collections.abc import Callable
 from urllib.parse import ParseResult, urlunparse
 
 from git import GitError
 from pydriller.git import Git
 
+from macaron.config.global_config import global_config
 from macaron.errors import CloneError, ConfigurationError, RepoCheckOutError
 from macaron.slsa_analyzer import git_url
 from macaron.slsa_analyzer.git_service.base_git_service import BaseGitService
@@ -36,10 +37,16 @@
 class GitLab(BaseGitService):
     """This class contains the spec of the GitLab service."""
 
-    def __init__(self, access_token_env_name: str) -> None:
-        """Initialize instance."""
+    def __init__(self, token_function: Callable[[], str]) -> None:
+        """Initialize instance.
+
+        Parameters
+        ----------
+        token_function: Callable[[], str]
+            A function that returns a token when called.
+        """
         super().__init__("gitlab")
-        self.access_token_env_name = access_token_env_name
+        self.token_function = token_function
 
     @abstractmethod
     def load_defaults(self) -> None:
@@ -81,7 +88,7 @@ def construct_clone_url(self, url: str) -> str:
 
         # Construct clone URL from ``urlparse`` result, with or without an access token.
         # https://docs.gitlab.com/ee/gitlab-basics/start-using-git.html#clone-using-a-token
-        access_token = os.environ.get(self.access_token_env_name)
+        access_token: str | None = self.token_function()
         if access_token:
             clone_url_netloc = f"oauth2:{access_token}@{self.hostname}"
         else:
@@ -227,7 +234,7 @@ class SelfHostedGitLab(GitLab):
 
     def __init__(self) -> None:
         """Initialize instance."""
-        super().__init__("MCN_SELF_HOSTED_GITLAB_TOKEN")
+        super().__init__(lambda: global_config.gl_self_host_token)
 
     def load_defaults(self) -> None:
         """Load the values for this git service from the ini configuration and environment variables.
@@ -248,10 +255,9 @@ def load_defaults(self) -> None:
         if not self.hostname:
             return
 
-        if not os.environ.get(self.access_token_env_name):
+        if not self.token_function():
             raise ConfigurationError(
-                f"Environment variable '{self.access_token_env_name}' is not set "
-                + f"for private GitLab service '{self.hostname}'."
+                f"Environment variable for self-hosted GitLab service '{self.hostname}' is not set."
             )
 
 
@@ -260,7 +266,7 @@ class PubliclyHostedGitLab(GitLab):
 
     def __init__(self) -> None:
         """Initialize instance."""
-        super().__init__("MCN_GITLAB_TOKEN")
+        super().__init__(lambda: global_config.gl_token)
 
     def load_defaults(self) -> None:
         """Load the values for this git service from the ini configuration and environment variables.

tests/slsa_analyzer/git_service/test_gitlab.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 """Tests for the GitLab git service."""
@@ -26,10 +26,11 @@
 )
 def test_construct_clone_url_without_token(repo_url: str) -> None:
     """Test if the ``construct_clone_url`` method produces proper clone URLs without the access token."""
-    clone_url = repo_url
-    gitlab = PubliclyHostedGitLab()
-    gitlab.load_defaults()
-    assert gitlab.construct_clone_url(repo_url) == clone_url
+    with mock.patch("macaron.config.global_config.global_config.gl_token", ""):
+        clone_url = repo_url
+        gitlab = PubliclyHostedGitLab()
+        gitlab.load_defaults()
+        assert gitlab.construct_clone_url(repo_url) == clone_url
 
 
 @pytest.mark.parametrize(
@@ -47,7 +48,7 @@ def test_construct_clone_url_without_token(repo_url: str) -> None:
 )
 def test_construct_clone_url_with_token(repo_url: str, clone_url: str) -> None:
     """Test if the ``construct_clone_url`` method produces proper clone URLs with the access token."""
-    with mock.patch.dict(os.environ, {"MCN_GITLAB_TOKEN": "abcxyz"}):
+    with mock.patch("macaron.config.global_config.global_config.gl_token", "abcxyz"):
         gitlab = PubliclyHostedGitLab()
         gitlab.load_defaults()
         assert gitlab.construct_clone_url(repo_url) == clone_url
@@ -88,7 +89,7 @@ def test_construct_clone_url_for_self_hosted_gitlab(
     # ``setup_test`` fixture.
     load_defaults(user_config_path)
 
-    with mock.patch.dict(os.environ, {"MCN_SELF_HOSTED_GITLAB_TOKEN": "abcxyz"}):
+    with mock.patch("macaron.config.global_config.global_config.gl_self_host_token", "abcxyz"):
         gitlab = SelfHostedGitLab()
         gitlab.load_defaults()
         assert gitlab.construct_clone_url(repo_url) == clone_url
@@ -108,8 +109,7 @@ def test_self_hosted_gitlab_without_env_set(tmp_path: Path) -> None:
     # pollution here, since we reload the ``defaults`` object before every test with the
     # ``setup_test`` fixture.
     load_defaults(user_config_path)
-
-    with mock.patch.dict(os.environ, {"MCN_SELF_HOSTED_GITLAB_TOKEN": ""}):
+    with mock.patch("macaron.config.global_config.global_config.gl_self_host_token", ""):
         gitlab = SelfHostedGitLab()
 
         with pytest.raises(ConfigurationError):
@@ -189,7 +189,7 @@ def test_origin_remote_url_masking(self_hosted_gitlab: Git, expected_origin_url:
     # ``setup_test`` fixture.
     load_defaults(user_config_path)
 
-    with mock.patch.dict(os.environ, {"MCN_SELF_HOSTED_GITLAB_TOKEN": "abcxyz"}):
+    with mock.patch("macaron.config.global_config.global_config.gl_self_host_token", "abcxyz"):
         gitlab = SelfHostedGitLab()
         gitlab.load_defaults()