Skip to content

Commit

Permalink
Consolidate linux and windows containerd config templates
Browse files Browse the repository at this point in the history 
Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
  • Loading branch information
@brandond
brandond committed Feb 4, 2025
1 parent f78769d commit 804b334
Show file tree
Hide file tree
Showing 7 changed files with 172 additions and 280 deletions.
55 changes: 21 additions & 34 deletions pkg/agent/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"strings"
"time"

"github.com/k3s-io/k3s/pkg/agent/containerd"
"github.com/k3s-io/k3s/pkg/agent/proxy"
agentutil "github.com/k3s-io/k3s/pkg/agent/util"
"github.com/k3s-io/k3s/pkg/cli/cmds"
Expand Down Expand Up @@ -647,43 +646,18 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
nodeConfig.Containerd.Config = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml")
nodeConfig.Containerd.Root = filepath.Join(envInfo.DataDir, "agent", "containerd")
nodeConfig.CRIDockerd.Root = filepath.Join(envInfo.DataDir, "agent", "cri-dockerd")
if !nodeConfig.Docker {
if nodeConfig.ImageServiceEndpoint != "" {
nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ImageServiceEndpoint
} else if nodeConfig.ContainerRuntimeEndpoint == "" {
switch nodeConfig.AgentConfig.Snapshotter {
case "overlayfs":
if err := containerd.OverlaySupported(nodeConfig.Containerd.Root); err != nil {
return nil, errors.Wrapf(err, "\"overlayfs\" snapshotter cannot be enabled for %q, try using \"fuse-overlayfs\" or \"native\"",
nodeConfig.Containerd.Root)
}
case "fuse-overlayfs":
if err := containerd.FuseoverlayfsSupported(nodeConfig.Containerd.Root); err != nil {
return nil, errors.Wrapf(err, "\"fuse-overlayfs\" snapshotter cannot be enabled for %q, try using \"native\"",
nodeConfig.Containerd.Root)
}
case "stargz":
if err := containerd.StargzSupported(nodeConfig.Containerd.Root); err != nil {
return nil, errors.Wrapf(err, "\"stargz\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"",
nodeConfig.Containerd.Root)
}
nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
}
} else {
nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ContainerRuntimeEndpoint
}
}
nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "agent", "containerd")
nodeConfig.Containerd.Log = filepath.Join(envInfo.DataDir, "agent", "containerd", "containerd.log")
nodeConfig.Containerd.Registry = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "certs.d")
nodeConfig.Containerd.NoDefault = envInfo.ContainerdNoDefault
nodeConfig.Containerd.NonrootDevices = envInfo.ContainerdNonrootDevices
nodeConfig.Containerd.Debug = envInfo.Debug
applyContainerdStateAndAddress(nodeConfig)
applyCRIDockerdAddress(nodeConfig)
applyContainerdQoSClassConfigFileIfPresent(envInfo, &nodeConfig.Containerd)
nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml.tmpl")

if nodeConfig.Docker {
} else {
}

if envInfo.BindAddress != "" {
nodeConfig.AgentConfig.ListenAddress = envInfo.BindAddress
} else {
Expand Down Expand Up @@ -739,13 +713,26 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
}
}

if nodeConfig.Docker {
if nodeConfig.ImageServiceEndpoint != "" {
nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ImageServiceEndpoint
}

if nodeConfig.ContainerRuntimeEndpoint != "" {
nodeConfig.AgentConfig.RuntimeSocket = nodeConfig.ContainerRuntimeEndpoint
} else if nodeConfig.Docker {
if err := applyCRIDockerdOSSpecificConfig(nodeConfig); err != nil {
return nil, err
}
nodeConfig.AgentConfig.CNIPlugin = true
nodeConfig.AgentConfig.RuntimeSocket = nodeConfig.CRIDockerd.Address
} else if nodeConfig.ContainerRuntimeEndpoint == "" {
nodeConfig.AgentConfig.RuntimeSocket = nodeConfig.Containerd.Address
} else {
nodeConfig.AgentConfig.RuntimeSocket = nodeConfig.ContainerRuntimeEndpoint
if err := applyContainerdOSSpecificConfig(nodeConfig); err != nil {
return nil, err
}
if err := applyContainerdQoSClassConfigFileIfPresent(envInfo, &nodeConfig.Containerd); err != nil {
return nil, err
}
nodeConfig.AgentConfig.RuntimeSocket = nodeConfig.Containerd.Address
}

if controlConfig.ClusterIPRange != nil {
Expand Down
42 changes: 36 additions & 6 deletions pkg/agent/config/config_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,47 @@
package config

import (
"errors"
"os"
"path/filepath"

"github.com/k3s-io/k3s/pkg/agent/containerd"
"github.com/k3s-io/k3s/pkg/cli/cmds"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)

func applyContainerdStateAndAddress(nodeConfig *config.Node) {
// applyContainerdOSSpecificConfig sets linux-specific containerd config
func applyContainerdOSSpecificConfig(nodeConfig *config.Node) error {
nodeConfig.Containerd.State = "/run/k3s/containerd"
nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock")
}

func applyCRIDockerdAddress(nodeConfig *config.Node) {
nodeConfig.CRIDockerd.Address = "unix:///run/k3s/cri-dockerd/cri-dockerd.sock"
// validate that the selected snapshotter supports the filesystem at the root path.
// for stargz, also overrides the image service endpoint path.
switch nodeConfig.AgentConfig.Snapshotter {
case "overlayfs":
if err := containerd.OverlaySupported(nodeConfig.Containerd.Root); err != nil {
return errors.Wrapf(err, "\"overlayfs\" snapshotter cannot be enabled for %q, try using \"fuse-overlayfs\" or \"native\"",
nodeConfig.Containerd.Root)
}
case "fuse-overlayfs":
if err := containerd.FuseoverlayfsSupported(nodeConfig.Containerd.Root); err != nil {
return errors.Wrapf(err, "\"fuse-overlayfs\" snapshotter cannot be enabled for %q, try using \"native\"",
nodeConfig.Containerd.Root)
}
case "stargz":
if err := containerd.StargzSupported(nodeConfig.Containerd.Root); err != nil {
return errors.Wrapf(err, "\"stargz\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"",
nodeConfig.Containerd.Root)
}
nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
}

return nil
}

func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, containerdConfig *config.Containerd) {
// applyContainerdQoSClassConfigFileIfPresent sets linux-specific qos config
func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, containerdConfig *config.Containerd) error {
containerdConfigDir := filepath.Join(envInfo.DataDir, "agent", "etc", "containerd")

blockioPath := filepath.Join(containerdConfigDir, "blockio_config.yaml")
Expand All @@ -44,6 +66,14 @@ func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, containerdC
containerdConfig.RDTConfig = rdtPath
}
}

return nil
}

// applyCRIDockerdOSSpecificConfig sets linux-specific cri-dockerd config
func applyCRIDockerdOSSpecificConfig(nodeConfig *config.Node) error {
nodeConfig.CRIDockerd.Address = "unix:///run/k3s/cri-dockerd/cri-dockerd.sock"
return nil
}

// configureACL will configure an Access Control List for the specified file.
Expand Down
18 changes: 13 additions & 5 deletions pkg/agent/config/config_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,25 @@ import (
"golang.org/x/sys/windows"
)

func applyContainerdStateAndAddress(nodeConfig *config.Node) {
// applyContainerdOSSpecificConfig sets windows-specific containerd config
func applyContainerdOSSpecificConfig(nodeConfig *config.Node) error {
nodeConfig.AgentConfig.Snapshotter = "windows"
nodeConfig.Containerd.State = filepath.Join(nodeConfig.Containerd.Root, "state")
nodeConfig.Containerd.Address = "npipe:////./pipe/containerd-containerd"
nodeConfig.DefaultRuntime = "runhcs-wcow-process"
return nil
}

func applyCRIDockerdAddress(nodeConfig *config.Node) {
nodeConfig.CRIDockerd.Address = "npipe:////.pipe/cri-dockerd"
// applyContainerdQoSClassConfigFileIfPresent sets windows-specific qos config
func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, containerdConfig *config.Containerd) error {
// QoS-class resource management not supported on windows.
return nil
}

func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, containerdConfig *config.Containerd) {
// QoS-class resource management not supported on windows.
// applyCRIDockerdOSpecificConfig sets windows-specific cri-dockerd config
func applyCRIDockerdOSpecificConfig(nodeConfig *config.Node) error {
nodeConfig.CRIDockerd.Address = "npipe:////.pipe/cri-dockerd"
return nil
}

// configureACL will configure an Access Control List for the specified file,
Expand Down
2 changes: 0 additions & 2 deletions pkg/agent/containerd/config_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@ func SetupContainerdConfig(cfg *config.Node) error {
containerdConfig := templates.ContainerdConfig{
NodeConfig: cfg,
DisableCgroup: true,
SystemdCgroup: false,
IsRunningInUserNS: false,
PrivateRegistryConfig: cfg.AgentConfig.Registry,
NoDefaultEndpoint: cfg.Containerd.NoDefault,
}
Expand Down
102 changes: 102 additions & 0 deletions pkg/agent/templates/templates.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,108 @@ type HostConfig struct {
Endpoints []RegistryEndpoint
}

const ContainerdConfigTemplate = `
{{- /* */ -}}
# File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead.
version = 2
root = {{ printf "%q" .NodeConfig.Containerd.Root }}
state = {{ printf "%q" .NodeConfig.Containerd.State }}
[plugins."io.containerd.internal.v1.opt"]
path = {{ printf "%q" .NodeConfig.Containerd.Opt }}
[plugins."io.containerd.grpc.v1.cri"]
stream_server_address = "127.0.0.1"
stream_server_port = "10010"
enable_selinux = {{ .NodeConfig.SELinux }}
enable_unprivileged_ports = {{ .EnableUnprivileged }}
enable_unprivileged_icmp = {{ .EnableUnprivileged }}
device_ownership_from_security_context = {{ .NonrootDevices }}
{{- if .DisableCgroup}}
disable_cgroup = true
{{end}}
{{- if .IsRunningInUserNS }}
disable_apparmor = true
restrict_oom_score_adj = true
{{end}}
{{- if .NodeConfig.AgentConfig.PauseImage }}
sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
{{end}}
{{- if .NodeConfig.AgentConfig.Snapshotter }}
[plugins."io.containerd.grpc.v1.cri".containerd]
snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}"
disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}}
{{ if .NodeConfig.DefaultRuntime }}default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}"{{end}}
{{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}
{{ if .NodeConfig.AgentConfig.ImageServiceSocket }}
[plugins."io.containerd.snapshotter.v1.stargz"]
cri_keychain_image_service_path = "{{ .NodeConfig.AgentConfig.ImageServiceSocket }}"
[plugins."io.containerd.snapshotter.v1.stargz".cri_keychain]
enable_keychain = true
{{end}}
[plugins."io.containerd.snapshotter.v1.stargz".registry]
config_path = "{{ .NodeConfig.Containerd.Registry }}"
{{ if .PrivateRegistryConfig }}
{{range $k, $v := .PrivateRegistryConfig.Configs }}
{{ if $v.Auth }}
[plugins."io.containerd.snapshotter.v1.stargz".registry.configs."{{$k}}".auth]
{{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}}
{{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}}
{{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
{{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
{{end}}
{{end}}
{{end}}
{{end}}
{{end}}
{{- if not .NodeConfig.NoFlannel }}
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
{{end}}
{{- if or .NodeConfig.Containerd.BlockIOConfig .NodeConfig.Containerd.RDTConfig }}
[plugins."io.containerd.service.v1.tasks-service"]
{{ if .NodeConfig.Containerd.BlockIOConfig }}blockio_config_file = "{{ .NodeConfig.Containerd.BlockIOConfig }}"{{end}}
{{ if .NodeConfig.Containerd.RDTConfig }}rdt_config_file = "{{ .NodeConfig.Containerd.RDTConfig }}"{{end}}
{{end}}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = {{ .SystemdCgroup }}
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "{{ .NodeConfig.Containerd.Registry }}"
{{ if .PrivateRegistryConfig }}
{{range $k, $v := .PrivateRegistryConfig.Configs }}
{{ if $v.Auth }}
[plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".auth]
{{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}}
{{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}}
{{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
{{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
{{end}}
{{end}}
{{end}}
{{range $k, $v := .ExtraRuntimes}}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."{{$k}}"]
runtime_type = "{{$v.RuntimeType}}"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."{{$k}}".options]
BinaryName = "{{$v.BinaryName}}"
SystemdCgroup = {{ $.SystemdCgroup }}
{{end}}
`

var HostsTomlHeader = "# File generated by " + version.Program + ". DO NOT EDIT.\n"

const HostsTomlTemplate = `
Expand Down
Loading

0 comments on commit 804b334

Please sign in to comment.