From 69e7d4f000a948676416e667230dc32c428686c2 Mon Sep 17 00:00:00 2001 From: Ryan Balfanz <133278+RyanBalfanz@users.noreply.github.com> Date: Thu, 7 Nov 2024 15:25:10 -0800 Subject: [PATCH] Add secrets fetcher tests where none existed before (#1019) * Add secrets fetcher sidecar tests * Add pyfakefs * fix AttributeError: module 'datetime' has no attribute 'UTC' on python < 3.11 * fix AssertionError: 'root' != 'vscode' on CI * clean up tests * Remove incomplete test `sets_mode` * Update tests/unit/sidecars/secrets_fetcher_tests.py Co-authored-by: Chris Kuehl * Use autospec=True when patching VaultClientFactory * Test setting file group ownership using the group database entry from the OS group ID --------- Co-authored-by: Chris Kuehl --- poetry.lock | 15 +- pyproject.toml | 1 + tests/unit/sidecars/secrets_fetcher_tests.py | 225 +++++++++++++++++++ 3 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 tests/unit/sidecars/secrets_fetcher_tests.py diff --git a/poetry.lock b/poetry.lock index 4d97e03a8..c3e5cc3a1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "advocate" @@ -2643,6 +2643,17 @@ snowballstemmer = ">=2.2.0" [package.extras] toml = ["tomli (>=1.2.3)"] +[[package]] +name = "pyfakefs" +version = "5.7.1" +description = "pyfakefs implements a fake file system that mocks the Python file system modules." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyfakefs-5.7.1-py3-none-any.whl", hash = "sha256:6503ffe7f401701cf974b502311f926da2b0657a72244a6ba36e985ceb3dd783"}, + {file = "pyfakefs-5.7.1.tar.gz", hash = "sha256:24774c632f3b67ea26fd56b08115ba7c339d5cd65655410bca8572d73a1ae9a4"}, +] + [[package]] name = "pygments" version = "2.18.0" @@ -4079,4 +4090,4 @@ zookeeper = ["kazoo"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "9c41fa9374f3163923ac7a9f244e197f4356adfd703cd1fe573d33739a1a2c37" +content-hash = "23b8a5451ce1b4c0f52f13e9e4a3cbff81a3e7ab2a10cfcb46206094194dc511" diff --git a/pyproject.toml b/pyproject.toml index c881d9988..3a9ad2ef5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,7 @@ webtest = "*" parameterized = "^0.9.0" opentelemetry-test-utils = "^0.47b0" ruff = "*" +pyfakefs = "^5.7.1" [tool.poetry.scripts] diff --git a/tests/unit/sidecars/secrets_fetcher_tests.py b/tests/unit/sidecars/secrets_fetcher_tests.py new file mode 100644 index 000000000..af8c1e4da --- /dev/null +++ b/tests/unit/sidecars/secrets_fetcher_tests.py @@ -0,0 +1,225 @@ +import configparser +import dataclasses +import datetime +import getpass +import grp +import io +import json +import os +import pathlib +import sys +import typing +import unittest.mock + +from pyfakefs.fake_filesystem_unittest import TestCase + +from baseplate.lib import config +from baseplate.sidecars import secrets_fetcher + +UTC: datetime.timezone +if sys.version_info > (3, 11): + UTC = datetime.UTC +else: + from datetime import timezone + + UTC = timezone.utc + +whoami = getpass.getuser() +group = grp.getgrgid(os.getgid()).gr_name + +configini = f""" +[secret-fetcher] +vault.url = https://vault.example.com:8200/ +vault.role = my-server-role +vault.auth_type = aws +vault.mount_point = aws-ec2 + +output.path = /var/local/secrets.json +output.owner = {whoami} +output.group = {group} +output.mode = 0400 + +secrets = + secret/one, + secret/two, + secret/three, + +callback = scripts/my-transformer # optional +""".strip() + + +@dataclasses.dataclass +class FakeVaultClient(secrets_fetcher.VaultClient): + token_expiration: datetime.datetime + token: str = "token" + + # @typing.override # TODO: Added in version 3.12. + def get_secret(self, secret_name: str) -> tuple[typing.Any, datetime.datetime]: + return secret_name.upper(), self.token_expiration + datetime.timedelta(seconds=30) + + +@dataclasses.dataclass +class FakeBadVaultClient(secrets_fetcher.VaultClient): + token_expiration: datetime.datetime + token: str = "token-bad" + + # @typing.override # TODO: Added in version 3.12. + def get_secret(self, secret_name: str) -> tuple[typing.Any, datetime.datetime]: + """Return a secret value that is not JSON serializable.""" + s = self.token_expiration + return s, self.token_expiration + datetime.timedelta(seconds=30) + + +class Tests(TestCase): + @classmethod + def setUpClass(cls): + spec = { + "vault": { + "url": config.DefaultFromEnv(config.String, "BASEPLATE_DEFAULT_VAULT_URL"), + "role": config.String, + "auth_type": config.Optional( + config.OneOf(**secrets_fetcher.VaultClientFactory.auth_types()), + default=secrets_fetcher.VaultClientFactory.auth_types()["aws"], + ), + "mount_point": config.DefaultFromEnv( + config.String, "BASEPLATE_VAULT_MOUNT_POINT", fallback="aws-ec2" + ), + }, + "output": { + "path": config.Optional(config.String, default="/var/local/secrets.json"), + "owner": config.Optional(config.UnixUser, default=0), + "group": config.Optional(config.UnixGroup, default=0), + "mode": config.Optional(config.Integer(base=8), default=0o400), # type: ignore + }, + "secrets": config.Optional(config.TupleOf(config.String), default=[]), + "callback": config.Optional(config.String), + } + + parser = configparser.RawConfigParser() + with io.StringIO(configini) as f: + parser.read_file(f) + fetcher_config = dict(parser.items("secret-fetcher")) + + cls.cfg = config.parse_config(fetcher_config, spec) + + def setUp(self): + self.setUpPyfakefs() + self.fake_fs().create_file("/var/local/secrets.json", contents="initial contents") + + cfg = self.cfg + now = datetime.datetime.now(UTC) + with unittest.mock.patch( + "baseplate.sidecars.secrets_fetcher.VaultClientFactory", + autospec=True, + ) as mock: + instance = mock.return_value + instance.get_client.return_value = FakeVaultClient(token_expiration=now) + f = secrets_fetcher.VaultClientFactory( + cfg.vault.url, cfg.vault.role, cfg.vault.auth_type, cfg.vault.mount_point + ) + secrets_fetcher.fetch_secrets(cfg, f) + + def test_is_file(self): + p = pathlib.Path("/var/local/secrets.json") + self.assertTrue(p.is_file()) + + def test_sets_owner(self): + p = pathlib.Path("/var/local/secrets.json") + self.assertEqual(p.owner(), whoami) + + def test_sets_group(self): + p = pathlib.Path("/var/local/secrets.json") + self.assertEqual(p.group(), group) + + def test_deletes_temporary_file(self): + p = pathlib.Path("/var/local/secrets.json" + ".tmp") + self.assertFalse(p.exists()) + + def test_text_contents(self): + p = pathlib.Path("/var/local/secrets.json") + text = p.read_text() + self.assertDictEqual( + json.loads(text), + { + "secrets": { + "secret/one": "SECRET/ONE", + "secret/two": "SECRET/TWO", + "secret/three": "SECRET/THREE", + }, + "vault": { + "token": "token", + "url": "https://vault.example.com:8200/", + }, + "vault_token": "token", + }, + ) + + +class BadJSONTests(TestCase): + @classmethod + def setUpClass(cls): + spec = { + "vault": { + "url": config.DefaultFromEnv(config.String, "BASEPLATE_DEFAULT_VAULT_URL"), + "role": config.String, + "auth_type": config.Optional( + config.OneOf(**secrets_fetcher.VaultClientFactory.auth_types()), + default=secrets_fetcher.VaultClientFactory.auth_types()["aws"], + ), + "mount_point": config.DefaultFromEnv( + config.String, "BASEPLATE_VAULT_MOUNT_POINT", fallback="aws-ec2" + ), + }, + "output": { + "path": config.Optional(config.String, default="/var/local/secrets.json"), + "owner": config.Optional(config.UnixUser, default=0), + "group": config.Optional(config.UnixGroup, default=0), + "mode": config.Optional(config.Integer(base=8), default=0o400), # type: ignore + }, + "secrets": config.Optional(config.TupleOf(config.String), default=[]), + "callback": config.Optional(config.String), + } + + parser = configparser.RawConfigParser() + with io.StringIO(configini) as f: + parser.read_file(f) + fetcher_config = dict(parser.items("secret-fetcher")) + + cls.cfg = config.parse_config(fetcher_config, spec) + + def setUp(self): + self.setUpPyfakefs() + self.fake_fs().create_file( + "/var/local/secrets.json", + contents="initial contents should remain unchanged", + ) + + cfg = self.cfg + now = datetime.datetime.now(UTC) + + with unittest.mock.patch("baseplate.sidecars.secrets_fetcher.VaultClientFactory") as mock: + instance = mock.return_value + instance.get_client.return_value = FakeBadVaultClient(token_expiration=now) + f = secrets_fetcher.VaultClientFactory( + cfg.vault.url, cfg.vault.role, cfg.vault.auth_type, cfg.vault.mount_point + ) + with self.assertRaises(TypeError): + secrets_fetcher.fetch_secrets(cfg, f) + + def test_temporary_file_is_not_deleted(self): + p = pathlib.Path("/var/local/secrets.json.tmp") + self.assertTrue(p.exists()) + + def test_temporary_file_is_partially_written(self): + p = pathlib.Path("/var/local/secrets.json.tmp") + text = p.read_text() + self.assertEqual(text, """{\n "secrets": {\n "secret/one": """) + + def test_secrets_file_exists(self): + p = pathlib.Path("/var/local/secrets.json") + self.assertTrue(p.exists()) + + def test_secrets_file_is_unchanged(self): + p = pathlib.Path("/var/local/secrets.json") + text = p.read_text() + self.assertEqual(text, """initial contents should remain unchanged""")