From fa87c3718b37e93d217cf9479684dd2439feed5b Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Mon, 9 Oct 2023 11:18:58 +0200 Subject: [PATCH 1/5] platform: change signature of RuntimeConf to get a pointer Signed-off-by: Mathieu Tortuyaux --- platform/cluster.go | 4 ++-- platform/machine/aws/machine.go | 2 +- platform/machine/azure/machine.go | 2 +- platform/machine/do/machine.go | 2 +- platform/machine/equinixmetal/machine.go | 2 +- platform/machine/esx/machine.go | 2 +- platform/machine/external/machine.go | 2 +- platform/machine/gcloud/machine.go | 2 +- platform/machine/openstack/machine.go | 2 +- platform/machine/qemu/machine.go | 2 +- platform/machine/unprivqemu/machine.go | 2 +- platform/platform.go | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/platform/cluster.go b/platform/cluster.go index e59b9dc50..e7adfc67b 100644 --- a/platform/cluster.go +++ b/platform/cluster.go @@ -283,8 +283,8 @@ func (bc *BaseCluster) Name() string { return bc.name } -func (bc *BaseCluster) RuntimeConf() RuntimeConfig { - return *bc.rconf +func (bc *BaseCluster) RuntimeConf() *RuntimeConfig { + return bc.rconf } func (bc *BaseCluster) ConsoleOutput() map[string]string { diff --git a/platform/machine/aws/machine.go b/platform/machine/aws/machine.go index bdd511972..d97392088 100644 --- a/platform/machine/aws/machine.go +++ b/platform/machine/aws/machine.go @@ -48,7 +48,7 @@ func (am *machine) PrivateIP() string { return *am.mach.PrivateIpAddress } -func (am *machine) RuntimeConf() platform.RuntimeConfig { +func (am *machine) RuntimeConf() *platform.RuntimeConfig { return am.cluster.RuntimeConf() } diff --git a/platform/machine/azure/machine.go b/platform/machine/azure/machine.go index 0a4c56fe4..2470f5dbb 100644 --- a/platform/machine/azure/machine.go +++ b/platform/machine/azure/machine.go @@ -45,7 +45,7 @@ func (am *machine) PrivateIP() string { return am.mach.PrivateIPAddress } -func (am *machine) RuntimeConf() platform.RuntimeConfig { +func (am *machine) RuntimeConf() *platform.RuntimeConfig { return am.cluster.RuntimeConf() } diff --git a/platform/machine/do/machine.go b/platform/machine/do/machine.go index dce8a4092..c449a2c32 100644 --- a/platform/machine/do/machine.go +++ b/platform/machine/do/machine.go @@ -44,7 +44,7 @@ func (dm *machine) PrivateIP() string { return dm.privateIP } -func (dm *machine) RuntimeConf() platform.RuntimeConfig { +func (dm *machine) RuntimeConf() *platform.RuntimeConfig { return dm.cluster.RuntimeConf() } diff --git a/platform/machine/equinixmetal/machine.go b/platform/machine/equinixmetal/machine.go index dda45855c..0a655db47 100644 --- a/platform/machine/equinixmetal/machine.go +++ b/platform/machine/equinixmetal/machine.go @@ -44,7 +44,7 @@ func (pm *machine) PrivateIP() string { return pm.privateIP } -func (pm *machine) RuntimeConf() platform.RuntimeConfig { +func (pm *machine) RuntimeConf() *platform.RuntimeConfig { return pm.cluster.RuntimeConf() } diff --git a/platform/machine/esx/machine.go b/platform/machine/esx/machine.go index ab175111c..2a7b04221 100644 --- a/platform/machine/esx/machine.go +++ b/platform/machine/esx/machine.go @@ -46,7 +46,7 @@ func (em *machine) PrivateIP() string { return em.mach.IPAddress } -func (em *machine) RuntimeConf() platform.RuntimeConfig { +func (em *machine) RuntimeConf() *platform.RuntimeConfig { return em.cluster.RuntimeConf() } diff --git a/platform/machine/external/machine.go b/platform/machine/external/machine.go index 58736f0c6..a2d6e16ea 100644 --- a/platform/machine/external/machine.go +++ b/platform/machine/external/machine.go @@ -40,7 +40,7 @@ func (pm *machine) PrivateIP() string { return pm.ipAddr } -func (pm *machine) RuntimeConf() platform.RuntimeConfig { +func (pm *machine) RuntimeConf() *platform.RuntimeConfig { return pm.cluster.RuntimeConf() } diff --git a/platform/machine/gcloud/machine.go b/platform/machine/gcloud/machine.go index ae7c605c8..00fd99a4e 100644 --- a/platform/machine/gcloud/machine.go +++ b/platform/machine/gcloud/machine.go @@ -45,7 +45,7 @@ func (gm *machine) PrivateIP() string { return gm.intIP } -func (gm *machine) RuntimeConf() platform.RuntimeConfig { +func (gm *machine) RuntimeConf() *platform.RuntimeConfig { return gm.gc.RuntimeConf() } diff --git a/platform/machine/openstack/machine.go b/platform/machine/openstack/machine.go index 823ae1afd..65a9bc6ef 100644 --- a/platform/machine/openstack/machine.go +++ b/platform/machine/openstack/machine.go @@ -103,7 +103,7 @@ func (om *machine) PrivateIP() string { return om.IP() } -func (om *machine) RuntimeConf() platform.RuntimeConfig { +func (om *machine) RuntimeConf() *platform.RuntimeConfig { return om.cluster.RuntimeConf() } diff --git a/platform/machine/qemu/machine.go b/platform/machine/qemu/machine.go index 7aa887d92..50883f734 100644 --- a/platform/machine/qemu/machine.go +++ b/platform/machine/qemu/machine.go @@ -46,7 +46,7 @@ func (m *machine) PrivateIP() string { return m.netif.DHCPv4[0].IP.String() } -func (m *machine) RuntimeConf() platform.RuntimeConfig { +func (m *machine) RuntimeConf() *platform.RuntimeConfig { return m.qc.RuntimeConf() } diff --git a/platform/machine/unprivqemu/machine.go b/platform/machine/unprivqemu/machine.go index 1eb8c6cd0..647f8f518 100644 --- a/platform/machine/unprivqemu/machine.go +++ b/platform/machine/unprivqemu/machine.go @@ -46,7 +46,7 @@ func (m *machine) PrivateIP() string { return m.privateAddr } -func (m *machine) RuntimeConf() platform.RuntimeConfig { +func (m *machine) RuntimeConf() *platform.RuntimeConfig { return m.qc.RuntimeConf() } diff --git a/platform/platform.go b/platform/platform.go index 1a53cc0fc..edef7b692 100644 --- a/platform/platform.go +++ b/platform/platform.go @@ -50,7 +50,7 @@ type Machine interface { PrivateIP() string // RuntimeConf returns the cluster's runtime configuration. - RuntimeConf() RuntimeConfig + RuntimeConf() *RuntimeConfig // SSHClient establishes a new SSH connection to the machine. SSHClient() (*ssh.Client, error) From 35f04370695a7e9daee3cd6a35d15bf87a30b732 Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Mon, 9 Oct 2023 11:22:31 +0200 Subject: [PATCH 2/5] platform: add RuntimeConf method to the Cluster interface Adding this method allows to access the runtime configuration in a test to get/set values based on some conditions. Signed-off-by: Mathieu Tortuyaux --- platform/platform.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/platform/platform.go b/platform/platform.go index edef7b692..dd9c01e28 100644 --- a/platform/platform.go +++ b/platform/platform.go @@ -113,6 +113,9 @@ type Cluster interface { // IgnitionVersion returns the version of Ignition supported by the // cluster IgnitionVersion() string + + // RuntimeConf returns a pointer to the runtime configuration. + RuntimeConf() *RuntimeConfig } // Flight represents a group of Clusters within a single platform. From 1d7ef28ff306282a20a48138d3ebe8040a5b4b3f Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Mon, 9 Oct 2023 11:10:04 +0200 Subject: [PATCH 3/5] kubeadm: add logic to enforce SELinux for Cilium CNI in Flatcar >= 3745 Signed-off-by: Mathieu Tortuyaux --- kola/tests/kubeadm/kubeadm.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/kola/tests/kubeadm/kubeadm.go b/kola/tests/kubeadm/kubeadm.go index 00c2595a6..e625427a0 100644 --- a/kola/tests/kubeadm/kubeadm.go +++ b/kola/tests/kubeadm/kubeadm.go @@ -368,6 +368,32 @@ func setup(c cluster.TestCluster, params map[string]interface{}) (platform.Machi return nil, fmt.Errorf("unable to create etcd node: %w", err) } + v := string(c.MustSSH(etcdNode, `set -euo pipefail; grep -m 1 "^VERSION=" /usr/lib/os-release | cut -d = -f 2`)) + if v == "" { + c.Fatalf("Assertion for version string failed") + } + + version, err := semver.NewVersion(v) + if err != nil { + c.Fatalf("unable to create semver version from %s: %v", version, err) + } + + // For Cilium CNI, we enforce SELinux only for version >= 3745 because the SELinux policies update (container_t/spc_t) is not yet + // propagated through all the channels. + // The etcd node will run with enforced SELinux anyway but we want to test SELinux on the worker / master nodes. + cni, ok := params["CNI"] + if !ok { + c.Fatal("unable to get CNI value") + } + + if cni == "cilium" && version.LessThan(semver.Version{Major: 3745}) { + r := c.RuntimeConf() + if r != nil { + plog.Infof("Setting SELinux to permissive mode") + r.NoEnableSelinux = true + } + } + if err := etcd.GetClusterHealth(c, etcdNode, 1); err != nil { return nil, fmt.Errorf("unable to get etcd node health: %w", err) } From ef60b06ad00d4ff5280cf8b490fc328815b88f55 Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Tue, 30 Aug 2022 15:34:01 +0200 Subject: [PATCH 4/5] kubeadm/cilium: patch Cilium daemon set This is required even with Permissive mode. Can be dropped once `spc_t` is supported on Flatcar. Picked-From: e8e97516601150c335ea0ea8961b04506dcdafb5 Signed-off-by: Mathieu Tortuyaux --- kola/tests/kubeadm/kubeadm.go | 8 ++++++-- kola/tests/kubeadm/templates.go | 1 + kola/tests/kubeadm/testdata/master-cilium-script.sh | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/kola/tests/kubeadm/kubeadm.go b/kola/tests/kubeadm/kubeadm.go index e625427a0..e703dc8fc 100644 --- a/kola/tests/kubeadm/kubeadm.go +++ b/kola/tests/kubeadm/kubeadm.go @@ -54,8 +54,12 @@ var ( _ = c.MustSSH(controller, "/opt/bin/cilium uninstall") version := params["CiliumVersion"].(string) cidr := params["PodSubnet"].(string) - cmd := fmt.Sprintf("/opt/bin/cilium install --config enable-endpoint-routes=true --config cluster-pool-ipv4-cidr=%s --version=%s --encryption=ipsec --wait --wait-duration 1m", cidr, version) - _ = c.MustSSH(controller, cmd) + cmd := fmt.Sprintf("/opt/bin/cilium install --config enable-endpoint-routes=true --config cluster-pool-ipv4-cidr=%s --version=%s --encryption=ipsec --wait=false --restart-unmanaged-pods=false --rollback=false", cidr, version) + _, _ = c.SSH(controller, cmd) + patch := `/opt/bin/kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}'` + _ = c.MustSSH(controller, patch) + status := "/opt/bin/cilium status --wait --wait-duration 1m" + _ = c.MustSSH(controller, status) }, }, }, diff --git a/kola/tests/kubeadm/templates.go b/kola/tests/kubeadm/templates.go index f54cfa990..45da1c44c 100644 --- a/kola/tests/kubeadm/templates.go +++ b/kola/tests/kubeadm/templates.go @@ -403,6 +403,7 @@ EOF --config enable-endpoint-routes=true \ --config cluster-pool-ipv4-cidr={{ .PodSubnet }} \ --version={{ .CiliumVersion }} 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT + kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}' # --wait will wait for status to report success /opt/bin/cilium status --wait 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT {{ end }} diff --git a/kola/tests/kubeadm/testdata/master-cilium-script.sh b/kola/tests/kubeadm/testdata/master-cilium-script.sh index 72c099e77..7964f8930 100644 --- a/kola/tests/kubeadm/testdata/master-cilium-script.sh +++ b/kola/tests/kubeadm/testdata/master-cilium-script.sh @@ -91,6 +91,7 @@ EOF --config enable-endpoint-routes=true \ --config cluster-pool-ipv4-cidr=192.168.0.0/17 \ --version=v0.11.1 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT + kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}' # --wait will wait for status to report success /opt/bin/cilium status --wait 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT From daead39903868f33811f9183e67caf5501e61b40 Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Mon, 9 Oct 2023 17:45:15 +0200 Subject: [PATCH 5/5] kubeadm: apply Cilium patch only when container_t is unavailable container_t brings 'spc_t' too which is required by Cilium. We patch the daemon-set only when the old label ('svirt_lxc_file_t') is detected. Signed-off-by: Mathieu Tortuyaux --- kola/tests/kubeadm/kubeadm.go | 2 +- kola/tests/kubeadm/templates.go | 2 +- kola/tests/kubeadm/testdata/master-cilium-script.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kola/tests/kubeadm/kubeadm.go b/kola/tests/kubeadm/kubeadm.go index e703dc8fc..e2aa0e9b2 100644 --- a/kola/tests/kubeadm/kubeadm.go +++ b/kola/tests/kubeadm/kubeadm.go @@ -56,7 +56,7 @@ var ( cidr := params["PodSubnet"].(string) cmd := fmt.Sprintf("/opt/bin/cilium install --config enable-endpoint-routes=true --config cluster-pool-ipv4-cidr=%s --version=%s --encryption=ipsec --wait=false --restart-unmanaged-pods=false --rollback=false", cidr, version) _, _ = c.SSH(controller, cmd) - patch := `/opt/bin/kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}'` + patch := `{ grep -q svirt_lxc_file_t /etc/selinux/mcs/contexts/lxc_contexts && /opt/bin/kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}'; } || true` _ = c.MustSSH(controller, patch) status := "/opt/bin/cilium status --wait --wait-duration 1m" _ = c.MustSSH(controller, status) diff --git a/kola/tests/kubeadm/templates.go b/kola/tests/kubeadm/templates.go index 45da1c44c..2ceae6d19 100644 --- a/kola/tests/kubeadm/templates.go +++ b/kola/tests/kubeadm/templates.go @@ -403,7 +403,7 @@ EOF --config enable-endpoint-routes=true \ --config cluster-pool-ipv4-cidr={{ .PodSubnet }} \ --version={{ .CiliumVersion }} 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT - kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}' + { grep -q svirt_lxc_file_t /etc/selinux/mcs/contexts/lxc_contexts && kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}'; } || true # --wait will wait for status to report success /opt/bin/cilium status --wait 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT {{ end }} diff --git a/kola/tests/kubeadm/testdata/master-cilium-script.sh b/kola/tests/kubeadm/testdata/master-cilium-script.sh index 7964f8930..61a1e6e5c 100644 --- a/kola/tests/kubeadm/testdata/master-cilium-script.sh +++ b/kola/tests/kubeadm/testdata/master-cilium-script.sh @@ -91,7 +91,7 @@ EOF --config enable-endpoint-routes=true \ --config cluster-pool-ipv4-cidr=192.168.0.0/17 \ --version=v0.11.1 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT - kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}' + { grep -q svirt_lxc_file_t /etc/selinux/mcs/contexts/lxc_contexts && kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}'; } || true # --wait will wait for status to report success /opt/bin/cilium status --wait 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT