diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index 54aaaff53..0ff07dc32 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -17,3 +17,4 @@ jobs: uses: golangci/golangci-lint-action@v3 with: version: v1.51.2 + args: --timeout=30m --config=.golangci.yaml diff --git a/.golangci.yaml b/.golangci.yaml index 9af9432e5..62a94f0b6 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -82,7 +82,7 @@ linters-settings: # Use sigs.k8s.io/kwok/pkg/utils/slices, which is a wrapper around slices - k8s.io/utils/strings/slices: "please use `sigs.k8s.io/kwok/pkg/utils/slices` instead" gocyclo: - min-complexity: 60 + min-complexity: 50 gosec: excludes: - G110 diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 88586fef6..9a52aba2b 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -74,3 +74,15 @@ const ( RuntimeTypePodman = "podman" RuntimeTypeBinary = "binary" ) + +// The following components is provided. +const ( + ComponentEtcd = "etcd" + ComponentKubeApiserver = "kube-apiserver" + ComponentKubeControllerManager = "kube-controller-manager" + ComponentKubeScheduler = "kube-scheduler" + ComponentKwokController = "kwok-controller" + ComponentDashboard = "dashboard" + ComponentPrometheus = "prometheus" + ComponentJaeger = "jaeger" +) diff --git a/pkg/kwok/server/debugging_exec.go b/pkg/kwok/server/debugging_exec.go index cda0f7bee..dc5b08f7b 100644 --- a/pkg/kwok/server/debugging_exec.go +++ b/pkg/kwok/server/debugging_exec.go @@ -104,11 +104,11 @@ func (s *Server) execInContainer(ctx context.Context, cmd []string, in io.Reader } func (s *Server) getExecTarget(podName, podNamespace string, containerName string) (*internalversion.ExecTarget, error) { - pf, has := slices.Find(s.execs.Get(), func(pf *internalversion.Exec) bool { + e, has := slices.Find(s.execs.Get(), func(pf *internalversion.Exec) bool { return pf.Name == podName && pf.Namespace == podNamespace }) if has { - exec, found := findContainerInExecs(containerName, pf.Spec.Execs) + exec, found := findContainerInExecs(containerName, e.Spec.Execs) if found { return exec, nil } diff --git a/pkg/kwokctl/components/dashboard.go b/pkg/kwokctl/components/dashboard.go index a4a8613e0..a5aa78ce7 100644 --- a/pkg/kwokctl/components/dashboard.go +++ b/pkg/kwokctl/components/dashboard.go @@ -18,6 +18,7 @@ package components import ( "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/utils/format" "sigs.k8s.io/kwok/pkg/utils/version" ) @@ -101,10 +102,10 @@ func BuildDashboardComponent(conf BuildDashboardComponentConfig) (component inte } component = internalversion.Component{ - Name: "dashboard", + Name: consts.ComponentDashboard, Image: conf.Image, Links: []string{ - "kube-apiserver", + consts.ComponentKubeApiserver, }, WorkDir: conf.Workdir, Ports: ports, diff --git a/pkg/kwokctl/components/etcd.go b/pkg/kwokctl/components/etcd.go index 188271bb1..9efe1af2f 100644 --- a/pkg/kwokctl/components/etcd.go +++ b/pkg/kwokctl/components/etcd.go @@ -20,6 +20,7 @@ import ( "runtime" "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" "sigs.k8s.io/kwok/pkg/utils/version" @@ -136,10 +137,10 @@ func BuildEtcdComponent(conf BuildEtcdComponentConfig) (component internalversio envs = append(envs, conf.ExtraEnvs...) return internalversion.Component{ - Name: "etcd", + Name: consts.ComponentEtcd, Version: conf.Version.String(), Volumes: volumes, - Command: []string{"etcd"}, + Command: []string{consts.ComponentEtcd}, Args: etcdArgs, Binary: conf.Binary, Ports: ports, diff --git a/pkg/kwokctl/components/jaeger.go b/pkg/kwokctl/components/jaeger.go index 671d854ce..dc8cc6fdc 100644 --- a/pkg/kwokctl/components/jaeger.go +++ b/pkg/kwokctl/components/jaeger.go @@ -18,6 +18,7 @@ package components import ( "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" "sigs.k8s.io/kwok/pkg/utils/net" @@ -70,7 +71,7 @@ func BuildJaegerComponent(conf BuildJaegerComponentConfig) (component internalve } return internalversion.Component{ - Name: "jaeger", + Name: consts.ComponentJaeger, Version: conf.Version.String(), Ports: ports, Volumes: volumes, diff --git a/pkg/kwokctl/components/kube_apiserver.go b/pkg/kwokctl/components/kube_apiserver.go index a399cdc1d..3bacb5c68 100644 --- a/pkg/kwokctl/components/kube_apiserver.go +++ b/pkg/kwokctl/components/kube_apiserver.go @@ -21,6 +21,7 @@ import ( "strings" "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" "sigs.k8s.io/kwok/pkg/utils/version" @@ -257,16 +258,16 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon envs := []internalversion.Env{} envs = append(envs, conf.ExtraEnvs...) - links := []string{"etcd"} + links := []string{consts.ComponentEtcd} if conf.TracingConfigPath != "" { - links = append(links, "jaeger") + links = append(links, consts.ComponentJaeger) } return internalversion.Component{ - Name: "kube-apiserver", + Name: consts.ComponentKubeApiserver, Version: conf.Version.String(), Links: links, - Command: []string{"kube-apiserver"}, + Command: []string{consts.ComponentKubeApiserver}, Ports: ports, Volumes: volumes, Args: kubeApiserverArgs, diff --git a/pkg/kwokctl/components/kube_controller_manager.go b/pkg/kwokctl/components/kube_controller_manager.go index 8cf6104da..f9e2c72d7 100644 --- a/pkg/kwokctl/components/kube_controller_manager.go +++ b/pkg/kwokctl/components/kube_controller_manager.go @@ -198,12 +198,12 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen envs = append(envs, conf.ExtraEnvs...) return internalversion.Component{ - Name: "kube-controller-manager", + Name: consts.ComponentKubeControllerManager, Version: conf.Version.String(), Links: []string{ - "kube-apiserver", + consts.ComponentKubeApiserver, }, - Command: []string{"kube-controller-manager"}, + Command: []string{consts.ComponentKubeControllerManager}, Volumes: volumes, Args: kubeControllerManagerArgs, Ports: ports, diff --git a/pkg/kwokctl/components/kube_scheduler.go b/pkg/kwokctl/components/kube_scheduler.go index a03ff042e..52bc333b4 100644 --- a/pkg/kwokctl/components/kube_scheduler.go +++ b/pkg/kwokctl/components/kube_scheduler.go @@ -188,12 +188,12 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon envs = append(envs, conf.ExtraEnvs...) return internalversion.Component{ - Name: "kube-scheduler", + Name: consts.ComponentKubeScheduler, Version: conf.Version.String(), Links: []string{ - "kube-apiserver", + consts.ComponentKubeApiserver, }, - Command: []string{"kube-scheduler"}, + Command: []string{consts.ComponentKubeScheduler}, Volumes: volumes, Args: kubeSchedulerArgs, Binary: conf.Binary, diff --git a/pkg/kwokctl/components/kwok_controller.go b/pkg/kwokctl/components/kwok_controller.go index 9f75b8192..3466d6dc5 100644 --- a/pkg/kwokctl/components/kwok_controller.go +++ b/pkg/kwokctl/components/kwok_controller.go @@ -18,6 +18,7 @@ package components import ( "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" "sigs.k8s.io/kwok/pkg/utils/version" @@ -124,10 +125,10 @@ func BuildKwokControllerComponent(conf BuildKwokControllerComponentConfig) (comp envs = append(envs, conf.ExtraEnvs...) return internalversion.Component{ - Name: "kwok-controller", + Name: consts.ComponentKwokController, Version: conf.Version.String(), Links: []string{ - "kube-apiserver", + consts.ComponentKubeApiserver, }, Ports: ports, Command: []string{"kwok"}, diff --git a/pkg/kwokctl/components/prometheus.go b/pkg/kwokctl/components/prometheus.go index 05b86217e..115a5a77a 100644 --- a/pkg/kwokctl/components/prometheus.go +++ b/pkg/kwokctl/components/prometheus.go @@ -18,6 +18,7 @@ package components import ( "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" "sigs.k8s.io/kwok/pkg/utils/version" @@ -92,16 +93,16 @@ func BuildPrometheusComponent(conf BuildPrometheusComponentConfig) (component in envs = append(envs, conf.ExtraEnvs...) return internalversion.Component{ - Name: "prometheus", + Name: consts.ComponentPrometheus, Version: conf.Version.String(), Links: []string{ - "etcd", - "kube-apiserver", - "kube-controller-manager", - "kube-scheduler", - "kwok-controller", + consts.ComponentEtcd, + consts.ComponentKubeApiserver, + consts.ComponentKubeControllerManager, + consts.ComponentKubeScheduler, + consts.ComponentKwokController, }, - Command: []string{"prometheus"}, + Command: []string{consts.ComponentPrometheus}, Ports: ports, Volumes: volumes, Args: prometheusArgs, diff --git a/pkg/kwokctl/runtime/binary/cluster.go b/pkg/kwokctl/runtime/binary/cluster.go index b9bbab6a9..e4b3be7b6 100644 --- a/pkg/kwokctl/runtime/binary/cluster.go +++ b/pkg/kwokctl/runtime/binary/cluster.go @@ -62,21 +62,17 @@ func (c *Cluster) Available(ctx context.Context) error { return nil } -func (c *Cluster) download(ctx context.Context) error { - config, err := c.Config(ctx) - if err != nil { - return err - } - conf := &config.Options +func (c *Cluster) download(ctx context.Context, env *env) error { + conf := &env.kwokctlConfig.Options - kubeApiserverPath := c.GetBinPath("kube-apiserver" + conf.BinSuffix) - err = c.DownloadWithCache(ctx, conf.CacheDir, conf.KubeApiserverBinary, kubeApiserverPath, 0750, conf.QuietPull) + kubeApiserverPath := c.GetBinPath(consts.ComponentKubeApiserver + conf.BinSuffix) + err := c.DownloadWithCache(ctx, conf.CacheDir, conf.KubeApiserverBinary, kubeApiserverPath, 0750, conf.QuietPull) if err != nil { return err } if !conf.DisableKubeControllerManager { - kubeControllerManagerPath := c.GetBinPath("kube-controller-manager" + conf.BinSuffix) + kubeControllerManagerPath := c.GetBinPath(consts.ComponentKubeControllerManager + conf.BinSuffix) err = c.DownloadWithCache(ctx, conf.CacheDir, conf.KubeControllerManagerBinary, kubeControllerManagerPath, 0750, conf.QuietPull) if err != nil { return err @@ -84,7 +80,7 @@ func (c *Cluster) download(ctx context.Context) error { } if !conf.DisableKubeScheduler { - kubeSchedulerPath := c.GetBinPath("kube-scheduler" + conf.BinSuffix) + kubeSchedulerPath := c.GetBinPath(consts.ComponentKubeScheduler + conf.BinSuffix) err = c.DownloadWithCache(ctx, conf.CacheDir, conf.KubeSchedulerBinary, kubeSchedulerPath, 0750, conf.QuietPull) if err != nil { return err @@ -93,22 +89,22 @@ func (c *Cluster) download(ctx context.Context) error { // TODO: Add dashboard binary // if conf.DashboardPort != 0 { - // kubeDashboardPath := c.GetBinPath("dashboard" + conf.BinSuffix) + // kubeDashboardPath := c.GetBinPath(consts.ComponentDashboard + conf.BinSuffix) // err = c.DownloadWithCache(ctx, conf.CacheDir, conf.DashboardBinary, kubeDashboardPath, 0750, conf.QuietPull) // if err != nil { // return err // } // } - kwokControllerPath := c.GetBinPath("kwok-controller" + conf.BinSuffix) + kwokControllerPath := c.GetBinPath(consts.ComponentKwokController + conf.BinSuffix) err = c.DownloadWithCache(ctx, conf.CacheDir, conf.KwokControllerBinary, kwokControllerPath, 0750, conf.QuietPull) if err != nil { return err } - etcdPath := c.GetBinPath("etcd" + conf.BinSuffix) + etcdPath := c.GetBinPath(consts.ComponentEtcd + conf.BinSuffix) if conf.EtcdBinary == "" { - err = c.DownloadWithCacheAndExtract(ctx, conf.CacheDir, conf.EtcdBinaryTar, etcdPath, "etcd"+conf.BinSuffix, 0750, conf.QuietPull, true) + err = c.DownloadWithCacheAndExtract(ctx, conf.CacheDir, conf.EtcdBinaryTar, etcdPath, consts.ComponentEtcd+conf.BinSuffix, 0750, conf.QuietPull, true) if err != nil { return err } @@ -120,9 +116,9 @@ func (c *Cluster) download(ctx context.Context) error { } if conf.PrometheusPort != 0 { - prometheusPath := c.GetBinPath("prometheus" + conf.BinSuffix) + prometheusPath := c.GetBinPath(consts.ComponentPrometheus + conf.BinSuffix) if conf.PrometheusBinary == "" { - err = c.DownloadWithCacheAndExtract(ctx, conf.CacheDir, conf.PrometheusBinaryTar, prometheusPath, "prometheus"+conf.BinSuffix, 0750, conf.QuietPull, true) + err = c.DownloadWithCacheAndExtract(ctx, conf.CacheDir, conf.PrometheusBinaryTar, prometheusPath, consts.ComponentPrometheus+conf.BinSuffix, 0750, conf.QuietPull, true) if err != nil { return err } @@ -152,12 +148,8 @@ func (c *Cluster) download(ctx context.Context) error { return nil } -func (c *Cluster) setup(ctx context.Context) error { - config, err := c.Config(ctx) - if err != nil { - return err - } - conf := &config.Options +func (c *Cluster) setup(ctx context.Context, env *env) error { + conf := &env.kwokctlConfig.Options pkiPath := c.GetWorkdirPath(runtime.PkiName) if !file.Exists(pkiPath) { @@ -184,7 +176,7 @@ func (c *Cluster) setup(ctx context.Context) error { if conf.KubeAuditPolicy != "" { auditLogPath := c.GetLogPath(runtime.AuditLogName) - err = c.CreateFile(auditLogPath) + err := c.CreateFile(auditLogPath) if err != nil { return err } @@ -197,7 +189,7 @@ func (c *Cluster) setup(ctx context.Context) error { } etcdDataPath := c.GetWorkdirPath(runtime.EtcdDataDirName) - err = c.MkdirAll(etcdDataPath) + err := c.MkdirAll(etcdDataPath) if err != nil { return fmt.Errorf("failed to mkdir etcd data path: %w", err) } @@ -218,6 +210,70 @@ func (c *Cluster) setupPorts(ctx context.Context, ports ...*uint32) error { return nil } +type env struct { + kwokctlConfig *internalversion.KwokctlConfiguration + verbosity log.Level + kubeconfigPath string + etcdDataPath string + kwokConfigPath string + pkiPath string + auditLogPath string + auditPolicyPath string + workdir string + caCertPath string + adminKeyPath string + adminCertPath string + scheme string +} + +func (c *Cluster) env(ctx context.Context) (*env, error) { + config, err := c.Config(ctx) + if err != nil { + return nil, err + } + + scheme := "http" + if config.Options.SecurePort { + scheme = "https" + } + + workdir := c.Workdir() + + kubeconfigPath := c.GetWorkdirPath(runtime.InHostKubeconfigName) + kwokConfigPath := c.GetWorkdirPath(runtime.ConfigName) + etcdDataPath := c.GetWorkdirPath(runtime.EtcdDataDirName) + pkiPath := c.GetWorkdirPath(runtime.PkiName) + caCertPath := path.Join(pkiPath, "ca.crt") + adminKeyPath := path.Join(pkiPath, "admin.key") + adminCertPath := path.Join(pkiPath, "admin.crt") + auditLogPath := "" + auditPolicyPath := "" + + if config.Options.KubeAuditPolicy != "" { + auditLogPath = c.GetLogPath(runtime.AuditLogName) + auditPolicyPath = c.GetWorkdirPath(runtime.AuditPolicyName) + } + + logger := log.FromContext(ctx) + verbosity := logger.Level() + + return &env{ + kwokctlConfig: config, + verbosity: verbosity, + kubeconfigPath: kubeconfigPath, + etcdDataPath: etcdDataPath, + kwokConfigPath: kwokConfigPath, + pkiPath: pkiPath, + auditLogPath: auditLogPath, + auditPolicyPath: auditPolicyPath, + workdir: workdir, + caCertPath: caCertPath, + adminKeyPath: adminKeyPath, + adminCertPath: adminCertPath, + scheme: scheme, + }, nil +} + // Install installs the cluster func (c *Cluster) Install(ctx context.Context) error { err := c.Cluster.Install(ctx) @@ -237,77 +293,95 @@ func (c *Cluster) Install(ctx context.Context) error { } } - logger := log.FromContext(ctx) - verbosity := logger.Level() - config, err := c.Config(ctx) + env, err := c.env(ctx) if err != nil { return err } - conf := &config.Options - err = c.download(ctx) + err = c.download(ctx, env) if err != nil { return err } - err = c.setup(ctx) + err = c.setup(ctx, env) if err != nil { return err } - scheme := "http" - if conf.SecurePort { - scheme = "https" + err = c.setupPorts(ctx, + &env.kwokctlConfig.Options.EtcdPeerPort, + &env.kwokctlConfig.Options.EtcdPort, + &env.kwokctlConfig.Options.KubeApiserverPort, + &env.kwokctlConfig.Options.KwokControllerPort, + ) + if err != nil { + return err } - workdir := c.Workdir() + err = c.addEtcd(ctx, env) + if err != nil { + return err + } - kubeconfigPath := c.GetWorkdirPath(runtime.InHostKubeconfigName) - kubeApiserverPath := c.GetBinPath("kube-apiserver" + conf.BinSuffix) - kubeControllerManagerPath := c.GetBinPath("kube-controller-manager" + conf.BinSuffix) - kubeSchedulerPath := c.GetBinPath("kube-scheduler" + conf.BinSuffix) - kwokControllerPath := c.GetBinPath("kwok-controller" + conf.BinSuffix) - kwokConfigPath := c.GetWorkdirPath(runtime.ConfigName) - etcdPath := c.GetBinPath("etcd" + conf.BinSuffix) - etcdDataPath := c.GetWorkdirPath(runtime.EtcdDataDirName) - pkiPath := c.GetWorkdirPath(runtime.PkiName) - caCertPath := path.Join(pkiPath, "ca.crt") - adminKeyPath := path.Join(pkiPath, "admin.key") - adminCertPath := path.Join(pkiPath, "admin.crt") - auditLogPath := "" - auditPolicyPath := "" + err = c.addKubeApiserver(ctx, env) + if err != nil { + return err + } - if conf.KubeAuditPolicy != "" { - auditLogPath = c.GetLogPath(runtime.AuditLogName) - auditPolicyPath = c.GetWorkdirPath(runtime.AuditPolicyName) + err = c.addKubeControllerManager(ctx, env) + if err != nil { + return err } - err = c.setupPorts(ctx, - &conf.EtcdPeerPort, - &conf.EtcdPort, - &conf.KubeApiserverPort, - &conf.KwokControllerPort, - ) + err = c.addKubeScheduler(ctx, env) if err != nil { return err } + err = c.addKwokController(ctx, env) + if err != nil { + return err + } + + err = c.addPrometheus(ctx, env) + if err != nil { + return err + } + + err = c.addJaeger(ctx, env) + if err != nil { + return err + } + + err = c.finishInstall(ctx, env) + if err != nil { + return err + } + + return nil +} + +func (c *Cluster) addEtcd(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options + // Configure the etcd + etcdPath := c.GetBinPath(consts.ComponentEtcd + conf.BinSuffix) + etcdVersion, err := c.ParseVersionFromBinary(ctx, etcdPath) if err != nil { return err } - etcdComponentPatches := runtime.GetComponentPatches(config, "etcd") + etcdComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentEtcd) etcdComponent, err := components.BuildEtcdComponent(components.BuildEtcdComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Binary: etcdPath, Version: etcdVersion, BindAddress: conf.BindAddress, - DataPath: etcdDataPath, + DataPath: env.etcdDataPath, Port: conf.EtcdPort, PeerPort: conf.EtcdPeerPort, - Verbosity: verbosity, + Verbosity: env.verbosity, ExtraArgs: etcdComponentPatches.ExtraArgs, ExtraVolumes: etcdComponentPatches.ExtraVolumes, ExtraEnvs: etcdComponentPatches.ExtraEnvs, @@ -315,9 +389,16 @@ func (c *Cluster) Install(ctx context.Context) error { if err != nil { return err } - config.Components = append(config.Components, etcdComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, etcdComponent) + return nil +} + +func (c *Cluster) addKubeApiserver(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options // Configure the kube-apiserver + kubeApiserverPath := c.GetBinPath(consts.ComponentKubeApiserver + conf.BinSuffix) + kubeApiserverVersion, err := c.ParseVersionFromBinary(ctx, kubeApiserverPath) if err != nil { return err @@ -346,9 +427,9 @@ func (c *Cluster) Install(ctx context.Context) error { } } - kubeApiserverComponentPatches := runtime.GetComponentPatches(config, "kube-apiserver") + kubeApiserverComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeApiserver) kubeApiserverComponent, err := components.BuildKubeApiserverComponent(components.BuildKubeApiserverComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Binary: kubeApiserverPath, Version: kubeApiserverVersion, BindAddress: conf.BindAddress, @@ -360,12 +441,12 @@ func (c *Cluster) Install(ctx context.Context) error { SecurePort: conf.SecurePort, KubeAuthorization: conf.KubeAuthorization, KubeAdmission: conf.KubeAdmission, - AuditPolicyPath: auditPolicyPath, - AuditLogPath: auditLogPath, - CaCertPath: caCertPath, - AdminCertPath: adminCertPath, - AdminKeyPath: adminKeyPath, - Verbosity: verbosity, + AuditPolicyPath: env.auditPolicyPath, + AuditLogPath: env.auditLogPath, + CaCertPath: env.caCertPath, + AdminCertPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, + Verbosity: env.verbosity, DisableQPSLimits: conf.DisableQPSLimits, TracingConfigPath: kubeApiserverTracingConfigPath, ExtraArgs: kubeApiserverComponentPatches.ExtraArgs, @@ -375,10 +456,17 @@ func (c *Cluster) Install(ctx context.Context) error { if err != nil { return err } - config.Components = append(config.Components, kubeApiserverComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kubeApiserverComponent) + return nil +} + +func (c *Cluster) addKubeControllerManager(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options // Configure the kube-controller-manager if !conf.DisableKubeControllerManager { + kubeControllerManagerPath := c.GetBinPath(consts.ComponentKubeControllerManager + conf.BinSuffix) + err = c.setupPorts(ctx, &conf.KubeControllerManagerPort, ) @@ -391,23 +479,23 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - kubeControllerManagerPatches := runtime.GetComponentPatches(config, "kube-controller-manager") + kubeControllerManagerPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeControllerManager) kubeControllerManagerComponent, err := components.BuildKubeControllerManagerComponent(components.BuildKubeControllerManagerComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Binary: kubeControllerManagerPath, Version: kubeControllerManagerVersion, BindAddress: conf.BindAddress, Port: conf.KubeControllerManagerPort, SecurePort: conf.SecurePort, - CaCertPath: caCertPath, - AdminCertPath: adminCertPath, - AdminKeyPath: adminKeyPath, + CaCertPath: env.caCertPath, + AdminCertPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, KubeAuthorization: conf.KubeAuthorization, - KubeconfigPath: kubeconfigPath, + KubeconfigPath: env.kubeconfigPath, KubeFeatureGates: conf.KubeFeatureGates, NodeMonitorPeriodMilliseconds: conf.KubeControllerManagerNodeMonitorPeriodMilliseconds, NodeMonitorGracePeriodMilliseconds: conf.KubeControllerManagerNodeMonitorGracePeriodMilliseconds, - Verbosity: verbosity, + Verbosity: env.verbosity, DisableQPSLimits: conf.DisableQPSLimits, ExtraArgs: kubeControllerManagerPatches.ExtraArgs, ExtraVolumes: kubeControllerManagerPatches.ExtraVolumes, @@ -416,15 +504,22 @@ func (c *Cluster) Install(ctx context.Context) error { if err != nil { return err } - config.Components = append(config.Components, kubeControllerManagerComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kubeControllerManagerComponent) } + return nil +} + +func (c *Cluster) addKubeScheduler(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options // Configure the kube-scheduler if !conf.DisableKubeScheduler { + kubeSchedulerPath := c.GetBinPath(consts.ComponentKubeScheduler + conf.BinSuffix) + schedulerConfigPath := "" if conf.KubeSchedulerConfig != "" { schedulerConfigPath = c.GetWorkdirPath(runtime.SchedulerConfigName) - err = c.CopySchedulerConfig(conf.KubeSchedulerConfig, schedulerConfigPath, kubeconfigPath) + err = c.CopySchedulerConfig(conf.KubeSchedulerConfig, schedulerConfigPath, env.kubeconfigPath) if err != nil { return err } @@ -442,21 +537,21 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - kubeSchedulerComponentPatches := runtime.GetComponentPatches(config, "kube-scheduler") + kubeSchedulerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeScheduler) kubeSchedulerComponent, err := components.BuildKubeSchedulerComponent(components.BuildKubeSchedulerComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Binary: kubeSchedulerPath, Version: kubeSchedulerVersion, BindAddress: conf.BindAddress, Port: conf.KubeSchedulerPort, SecurePort: conf.SecurePort, - CaCertPath: caCertPath, - AdminCertPath: adminCertPath, - AdminKeyPath: adminKeyPath, + CaCertPath: env.caCertPath, + AdminCertPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, ConfigPath: schedulerConfigPath, - KubeconfigPath: kubeconfigPath, + KubeconfigPath: env.kubeconfigPath, KubeFeatureGates: conf.KubeFeatureGates, - Verbosity: verbosity, + Verbosity: env.verbosity, DisableQPSLimits: conf.DisableQPSLimits, ExtraArgs: kubeSchedulerComponentPatches.ExtraArgs, ExtraVolumes: kubeSchedulerComponentPatches.ExtraVolumes, @@ -465,30 +560,37 @@ func (c *Cluster) Install(ctx context.Context) error { if err != nil { return err } - config.Components = append(config.Components, kubeSchedulerComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kubeSchedulerComponent) } + return nil +} + +func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options // Configure the kwok-controller + kwokControllerPath := c.GetBinPath(consts.ComponentKwokController + conf.BinSuffix) + kwokControllerVersion, err := c.ParseVersionFromBinary(ctx, kwokControllerPath) if err != nil { return err } - kwokControllerComponentPatches := runtime.GetComponentPatches(config, "kwok-controller") + kwokControllerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKwokController) kwokControllerComponent := components.BuildKwokControllerComponent(components.BuildKwokControllerComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Binary: kwokControllerPath, Version: kwokControllerVersion, BindAddress: conf.BindAddress, Port: conf.KwokControllerPort, - ConfigPath: kwokConfigPath, - KubeconfigPath: kubeconfigPath, - CaCertPath: caCertPath, - AdminCertPath: adminCertPath, - AdminKeyPath: adminKeyPath, + ConfigPath: env.kwokConfigPath, + KubeconfigPath: env.kubeconfigPath, + CaCertPath: env.caCertPath, + AdminCertPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, NodeName: "localhost", - Verbosity: verbosity, + Verbosity: env.verbosity, NodeLeaseDurationSeconds: conf.NodeLeaseDurationSeconds, ExtraArgs: kwokControllerComponentPatches.ExtraArgs, ExtraEnvs: kwokControllerComponentPatches.ExtraEnvs, @@ -496,17 +598,22 @@ func (c *Cluster) Install(ctx context.Context) error { if err != nil { return err } - config.Components = append(config.Components, kwokControllerComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kwokControllerComponent) + return nil +} + +func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options // Configure the prometheus if conf.PrometheusPort != 0 { - prometheusPath := c.GetBinPath("prometheus" + conf.BinSuffix) + prometheusPath := c.GetBinPath(consts.ComponentPrometheus + conf.BinSuffix) prometheusData, err := BuildPrometheus(BuildPrometheusConfig{ ProjectName: c.Name(), SecurePort: conf.SecurePort, - AdminCrtPath: adminCertPath, - AdminKeyPath: adminKeyPath, + AdminCrtPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, PrometheusPort: conf.PrometheusPort, EtcdPort: conf.EtcdPort, KubeApiserverPort: conf.KubeApiserverPort, @@ -528,15 +635,15 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - prometheusComponentPatches := runtime.GetComponentPatches(config, "prometheus") + prometheusComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentPrometheus) prometheusComponent, err := components.BuildPrometheusComponent(components.BuildPrometheusComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Binary: prometheusPath, Version: prometheusVersion, BindAddress: conf.BindAddress, Port: conf.PrometheusPort, ConfigPath: prometheusConfigPath, - Verbosity: verbosity, + Verbosity: env.verbosity, ExtraArgs: prometheusComponentPatches.ExtraArgs, ExtraVolumes: prometheusComponentPatches.ExtraVolumes, ExtraEnvs: prometheusComponentPatches.ExtraEnvs, @@ -544,8 +651,13 @@ func (c *Cluster) Install(ctx context.Context) error { if err != nil { return err } - config.Components = append(config.Components, prometheusComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, prometheusComponent) } + return nil +} + +func (c *Cluster) addJaeger(ctx context.Context, env *env) error { + conf := &env.kwokctlConfig.Options // Configure the jaeger if conf.JaegerPort != 0 { @@ -556,49 +668,54 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - jaegerComponentPatches := runtime.GetComponentPatches(config, "jaeger") + jaegerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentJaeger) jaegerComponent, err := components.BuildJaegerComponent(components.BuildJaegerComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Binary: jaegerPath, Version: jaegerVersion, BindAddress: conf.BindAddress, Port: conf.JaegerPort, OtlpGrpcPort: conf.JaegerOtlpGrpcPort, - Verbosity: verbosity, + Verbosity: env.verbosity, ExtraArgs: jaegerComponentPatches.ExtraArgs, ExtraVolumes: jaegerComponentPatches.ExtraVolumes, }) if err != nil { return err } - config.Components = append(config.Components, jaegerComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, jaegerComponent) } + return nil +} + +func (c *Cluster) finishInstall(ctx context.Context, env *env) error { + conf := &env.kwokctlConfig.Options // Setup kubeconfig kubeconfigData, err := kubeconfig.EncodeKubeconfig(kubeconfig.BuildKubeconfig(kubeconfig.BuildKubeconfigConfig{ ProjectName: c.Name(), SecurePort: conf.SecurePort, - Address: scheme + "://" + net.LocalAddress + ":" + format.String(conf.KubeApiserverPort), - CACrtPath: caCertPath, - AdminCrtPath: adminCertPath, - AdminKeyPath: adminKeyPath, + Address: env.scheme + "://" + net.LocalAddress + ":" + format.String(conf.KubeApiserverPort), + CACrtPath: env.caCertPath, + AdminCrtPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, })) if err != nil { return err } - err = c.WriteFile(kubeconfigPath, kubeconfigData) + err = c.WriteFile(env.kubeconfigPath, kubeconfigData) if err != nil { return err } // Save config - err = c.SetConfig(ctx, config) + err = c.SetConfig(ctx, env.kwokctlConfig) if err != nil { - logger.Error("Failed to set config", err) + return err } err = c.Save(ctx) if err != nil { - logger.Error("Failed to update cluster", err) + return err } return nil diff --git a/pkg/kwokctl/runtime/binary/cluster_snapshot.go b/pkg/kwokctl/runtime/binary/cluster_snapshot.go index b575ddb67..6b6540db2 100644 --- a/pkg/kwokctl/runtime/binary/cluster_snapshot.go +++ b/pkg/kwokctl/runtime/binary/cluster_snapshot.go @@ -19,6 +19,7 @@ package binary import ( "context" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/kwokctl/runtime" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/wait" @@ -38,12 +39,12 @@ func (c *Cluster) SnapshotSave(ctx context.Context, path string) error { func (c *Cluster) SnapshotRestore(ctx context.Context, path string) error { logger := log.FromContext(ctx) - err := c.StopComponent(ctx, "etcd") + err := c.StopComponent(ctx, consts.ComponentEtcd) if err != nil { logger.Error("Failed to stop etcd", err) } defer func() { - err = c.StartComponent(ctx, "etcd") + err = c.StartComponent(ctx, consts.ComponentEtcd) if err != nil { logger.Error("Failed to start etcd", err) } @@ -85,11 +86,11 @@ func (c *Cluster) SnapshotSaveWithYAML(ctx context.Context, path string, filters func (c *Cluster) SnapshotRestoreWithYAML(ctx context.Context, path string, filters []string) error { logger := log.FromContext(ctx) err := wait.Poll(ctx, func(ctx context.Context) (bool, error) { - err := c.StopComponent(ctx, "kube-controller-manager") + err := c.StopComponent(ctx, consts.ComponentKubeControllerManager) if err != nil { return false, err } - component, err := c.GetComponent(ctx, "kube-controller-manager") + component, err := c.GetComponent(ctx, consts.ComponentKubeControllerManager) if err != nil { return false, err } @@ -100,7 +101,7 @@ func (c *Cluster) SnapshotRestoreWithYAML(ctx context.Context, path string, filt logger.Error("Failed to stop kube-controller-manager", err) } defer func() { - err = c.StartComponent(ctx, "kube-controller-manager") + err = c.StartComponent(ctx, consts.ComponentKubeControllerManager) if err != nil { logger.Error("Failed to start kube-controller-manager", err) } diff --git a/pkg/kwokctl/runtime/compose/cluster.go b/pkg/kwokctl/runtime/compose/cluster.go index df56a4e13..1c8bb9809 100644 --- a/pkg/kwokctl/runtime/compose/cluster.go +++ b/pkg/kwokctl/runtime/compose/cluster.go @@ -25,6 +25,7 @@ import ( "strings" "time" + "sigs.k8s.io/kwok/pkg/apis/internalversion" "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/kwokctl/components" "sigs.k8s.io/kwok/pkg/kwokctl/dryrun" @@ -130,12 +131,8 @@ func (c *Cluster) Available(ctx context.Context) error { return c.Exec(ctx, c.runtime, "version") } -func (c *Cluster) pullAllImages(ctx context.Context) error { - config, err := c.Config(ctx) - if err != nil { - return err - } - conf := &config.Options +func (c *Cluster) pullAllImages(ctx context.Context, env *env) error { + conf := &env.kwokctlConfig.Options images := []string{ conf.EtcdImage, conf.KubeApiserverImage, @@ -156,22 +153,16 @@ func (c *Cluster) pullAllImages(ctx context.Context) error { if conf.JaegerPort != 0 { images = append(images, conf.JaegerImage) } - err = c.PullImages(ctx, c.runtime, images, conf.QuietPull) + err := c.PullImages(ctx, c.runtime, images, conf.QuietPull) if err != nil { return err } return nil } -func (c *Cluster) setup(ctx context.Context) error { - config, err := c.Config(ctx) - if err != nil { - return err - } - conf := &config.Options - - pkiPath := c.GetWorkdirPath(runtime.PkiName) - if !file.Exists(pkiPath) { +func (c *Cluster) setup(ctx context.Context, env *env) error { + conf := &env.kwokctlConfig.Options + if !file.Exists(env.pkiPath) { sans := []string{ c.Name() + "-kube-apiserver", } @@ -185,37 +176,34 @@ func (c *Cluster) setup(ctx context.Context) error { if len(conf.KubeApiserverCertSANs) != 0 { sans = append(sans, conf.KubeApiserverCertSANs...) } - err = c.MkdirAll(pkiPath) + err = c.MkdirAll(env.pkiPath) if err != nil { return fmt.Errorf("failed to create pki dir: %w", err) } - err = c.GeneratePki(pkiPath, sans...) + err = c.GeneratePki(env.pkiPath, sans...) if err != nil { return fmt.Errorf("failed to generate pki: %w", err) } } if conf.KubeAuditPolicy != "" { - err = c.MkdirAll(c.GetWorkdirPath("logs")) + err := c.MkdirAll(c.GetWorkdirPath("logs")) if err != nil { return err } - auditLogPath := c.GetLogPath(runtime.AuditLogName) - err = c.CreateFile(auditLogPath) + err = c.CreateFile(env.auditLogPath) if err != nil { return err } - auditPolicyPath := c.GetWorkdirPath(runtime.AuditPolicyName) - err = c.CopyFile(conf.KubeAuditPolicy, auditPolicyPath) + err = c.CopyFile(conf.KubeAuditPolicy, env.auditPolicyPath) if err != nil { return err } } - etcdDataPath := c.GetWorkdirPath(runtime.EtcdDataDirName) - err = c.MkdirAll(etcdDataPath) + err := c.MkdirAll(env.etcdDataPath) if err != nil { return fmt.Errorf("failed to mkdir etcd data path: %w", err) } @@ -236,24 +224,33 @@ func (c *Cluster) setupPorts(ctx context.Context, ports ...*uint32) error { return nil } -// Install installs the cluster -func (c *Cluster) Install(ctx context.Context) error { - err := c.Cluster.Install(ctx) - if err != nil { - return err - } +type env struct { + kwokctlConfig *internalversion.KwokctlConfiguration + verbosity log.Level + inClusterOnHostKubeconfigPath string + inClusterKubeconfig string + kubeconfigPath string + etcdDataPath string + kwokConfigPath string + pkiPath string + auditLogPath string + auditPolicyPath string + workdir string + caCertPath string + adminKeyPath string + adminCertPath string + inClusterPkiPath string + inClusterCaCertPath string + inClusterAdminKeyPath string + inClusterAdminCertPath string + inClusterPort uint32 + scheme string +} - logger := log.FromContext(ctx) - verbosity := logger.Level() +func (c *Cluster) env(ctx context.Context) (*env, error) { config, err := c.Config(ctx) if err != nil { - return err - } - conf := &config.Options - - err = c.setup(ctx) - if err != nil { - return err + return nil, err } inClusterOnHostKubeconfigPath := c.GetWorkdirPath(runtime.InClusterKubeconfigName) @@ -264,7 +261,7 @@ func (c *Cluster) Install(ctx context.Context) error { pkiPath := c.GetWorkdirPath(runtime.PkiName) auditLogPath := "" auditPolicyPath := "" - if conf.KubeAuditPolicy != "" { + if config.Options.KubeAuditPolicy != "" { auditLogPath = c.GetLogPath(runtime.AuditLogName) auditPolicyPath = c.GetWorkdirPath(runtime.AuditPolicyName) } @@ -280,42 +277,137 @@ func (c *Cluster) Install(ctx context.Context) error { inClusterPort := uint32(8080) scheme := "http" - if conf.SecurePort { + if config.Options.SecurePort { scheme = "https" inClusterPort = 6443 } + logger := log.FromContext(ctx) + verbosity := logger.Level() + + return &env{ + kwokctlConfig: config, + verbosity: verbosity, + inClusterOnHostKubeconfigPath: inClusterOnHostKubeconfigPath, + inClusterKubeconfig: inClusterKubeconfig, + kubeconfigPath: kubeconfigPath, + etcdDataPath: etcdDataPath, + kwokConfigPath: kwokConfigPath, + pkiPath: pkiPath, + auditLogPath: auditLogPath, + auditPolicyPath: auditPolicyPath, + workdir: workdir, + caCertPath: caCertPath, + adminKeyPath: adminKeyPath, + adminCertPath: adminCertPath, + inClusterPkiPath: inClusterPkiPath, + inClusterCaCertPath: inClusterCaCertPath, + inClusterAdminKeyPath: inClusterAdminKeyPath, + inClusterAdminCertPath: inClusterAdminCertPath, + inClusterPort: inClusterPort, + scheme: scheme, + }, nil +} + +// Install installs the cluster +func (c *Cluster) Install(ctx context.Context) error { + err := c.Cluster.Install(ctx) + if err != nil { + return err + } + + env, err := c.env(ctx) + if err != nil { + return err + } + + err = c.setup(ctx, env) + if err != nil { + return err + } + err = c.setupPorts(ctx, - &conf.KubeApiserverPort, + &env.kwokctlConfig.Options.KubeApiserverPort, ) if err != nil { return err } - err = c.pullAllImages(ctx) + err = c.pullAllImages(ctx, env) if err != nil { return err } + err = c.addEtcd(ctx, env) + if err != nil { + return err + } + + err = c.addKubeApiserver(ctx, env) + if err != nil { + return err + } + + err = c.addKubeControllerManager(ctx, env) + if err != nil { + return err + } + + err = c.addKubeScheduler(ctx, env) + if err != nil { + return err + } + + err = c.addKwokController(ctx, env) + if err != nil { + return err + } + + err = c.addPrometheus(ctx, env) + if err != nil { + return err + } + + err = c.addJaeger(ctx, env) + if err != nil { + return err + } + + err = c.addDashboard(ctx, env) + if err != nil { + return err + } + + err = c.finishInstall(ctx, env) + if err != nil { + return err + } + + return nil +} + +func (c *Cluster) addEtcd(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options + // Configure the etcd etcdVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.EtcdImage, "etcd") if err != nil { return err } - etcdComponentPatches := runtime.GetComponentPatches(config, "etcd") + etcdComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentEtcd) etcdComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(etcdComponentPatches.ExtraVolumes) if err != nil { return fmt.Errorf("failed to expand host volumes for etcd component: %w", err) } etcdComponent, err := components.BuildEtcdComponent(components.BuildEtcdComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Image: conf.EtcdImage, Version: etcdVersion, BindAddress: net.PublicAddress, Port: conf.EtcdPort, - DataPath: etcdDataPath, - Verbosity: verbosity, + DataPath: env.etcdDataPath, + Verbosity: env.verbosity, ExtraArgs: etcdComponentPatches.ExtraArgs, ExtraVolumes: etcdComponentPatches.ExtraVolumes, ExtraEnvs: etcdComponentPatches.ExtraEnvs, @@ -323,15 +415,20 @@ func (c *Cluster) Install(ctx context.Context) error { if err != nil { return err } - config.Components = append(config.Components, etcdComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, etcdComponent) + return nil +} + +func (c *Cluster) addKubeApiserver(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options // Configure the kube-apiserver - kubeApiserverVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.KubeApiserverImage, "kube-apiserver") + kubeApiserverVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.KubeApiserverImage, consts.ComponentKubeApiserver) if err != nil { return err } - kubeApiserverComponentPatches := runtime.GetComponentPatches(config, "kube-apiserver") + kubeApiserverComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeApiserver) kubeApiserverComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(kubeApiserverComponentPatches.ExtraVolumes) if err != nil { return fmt.Errorf("failed to expand host volumes for kube api server component: %w", err) @@ -354,7 +451,7 @@ func (c *Cluster) Install(ctx context.Context) error { } kubeApiserverComponent, err := components.BuildKubeApiserverComponent(components.BuildKubeApiserverComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Image: conf.KubeApiserverImage, Version: kubeApiserverVersion, BindAddress: net.PublicAddress, @@ -364,14 +461,14 @@ func (c *Cluster) Install(ctx context.Context) error { SecurePort: conf.SecurePort, KubeAuthorization: conf.KubeAuthorization, KubeAdmission: conf.KubeAdmission, - AuditPolicyPath: auditPolicyPath, - AuditLogPath: auditLogPath, - CaCertPath: caCertPath, - AdminCertPath: adminCertPath, - AdminKeyPath: adminKeyPath, + AuditPolicyPath: env.auditPolicyPath, + AuditLogPath: env.auditLogPath, + CaCertPath: env.caCertPath, + AdminCertPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, EtcdPort: conf.EtcdPort, EtcdAddress: c.Name() + "-etcd", - Verbosity: verbosity, + Verbosity: env.verbosity, DisableQPSLimits: conf.DisableQPSLimits, TracingConfigPath: kubeApiserverTracingConfigPath, ExtraArgs: kubeApiserverComponentPatches.ExtraArgs, @@ -381,34 +478,39 @@ func (c *Cluster) Install(ctx context.Context) error { if err != nil { return err } - config.Components = append(config.Components, kubeApiserverComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kubeApiserverComponent) + return nil +} + +func (c *Cluster) addKubeControllerManager(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options // Configure the kube-controller-manager if !conf.DisableKubeControllerManager { - kubeControllerManagerVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.KubeControllerManagerImage, "kube-controller-manager") + kubeControllerManagerVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.KubeControllerManagerImage, consts.ComponentKubeControllerManager) if err != nil { return err } - kubeControllerManagerComponentPatches := runtime.GetComponentPatches(config, "kube-controller-manager") + kubeControllerManagerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeControllerManager) kubeControllerManagerComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(kubeControllerManagerComponentPatches.ExtraVolumes) if err != nil { return fmt.Errorf("failed to expand host volumes for kube controller manager component: %w", err) } kubeControllerManagerComponent, err := components.BuildKubeControllerManagerComponent(components.BuildKubeControllerManagerComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Image: conf.KubeControllerManagerImage, Version: kubeControllerManagerVersion, BindAddress: net.PublicAddress, Port: conf.KubeControllerManagerPort, SecurePort: conf.SecurePort, - CaCertPath: caCertPath, - AdminCertPath: adminCertPath, - AdminKeyPath: adminKeyPath, + CaCertPath: env.caCertPath, + AdminCertPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, KubeAuthorization: conf.KubeAuthorization, - KubeconfigPath: inClusterOnHostKubeconfigPath, + KubeconfigPath: env.inClusterOnHostKubeconfigPath, KubeFeatureGates: conf.KubeFeatureGates, - Verbosity: verbosity, + Verbosity: env.verbosity, DisableQPSLimits: conf.DisableQPSLimits, NodeMonitorPeriodMilliseconds: conf.KubeControllerManagerNodeMonitorPeriodMilliseconds, NodeMonitorGracePeriodMilliseconds: conf.KubeControllerManagerNodeMonitorGracePeriodMilliseconds, @@ -419,44 +521,49 @@ func (c *Cluster) Install(ctx context.Context) error { if err != nil { return err } - config.Components = append(config.Components, kubeControllerManagerComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kubeControllerManagerComponent) } + return nil +} + +func (c *Cluster) addKubeScheduler(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options // Configure the kube-scheduler if !conf.DisableKubeScheduler { schedulerConfigPath := "" if conf.KubeSchedulerConfig != "" { schedulerConfigPath = c.GetWorkdirPath(runtime.SchedulerConfigName) - err = c.CopySchedulerConfig(conf.KubeSchedulerConfig, schedulerConfigPath, inClusterKubeconfig) + err = c.CopySchedulerConfig(conf.KubeSchedulerConfig, schedulerConfigPath, env.inClusterKubeconfig) if err != nil { return err } } - kubeSchedulerVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.KubeSchedulerImage, "kube-scheduler") + kubeSchedulerVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.KubeSchedulerImage, consts.ComponentKubeScheduler) if err != nil { return err } - kubeSchedulerComponentPatches := runtime.GetComponentPatches(config, "kube-scheduler") + kubeSchedulerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeScheduler) kubeSchedulerComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(kubeSchedulerComponentPatches.ExtraVolumes) if err != nil { return fmt.Errorf("failed to expand host volumes for kube scheduler component: %w", err) } kubeSchedulerComponent, err := components.BuildKubeSchedulerComponent(components.BuildKubeSchedulerComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Image: conf.KubeSchedulerImage, Version: kubeSchedulerVersion, BindAddress: net.PublicAddress, Port: conf.KubeSchedulerPort, SecurePort: conf.SecurePort, - CaCertPath: caCertPath, - AdminCertPath: adminCertPath, - AdminKeyPath: adminKeyPath, + CaCertPath: env.caCertPath, + AdminCertPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, ConfigPath: schedulerConfigPath, - KubeconfigPath: inClusterOnHostKubeconfigPath, + KubeconfigPath: env.inClusterOnHostKubeconfigPath, KubeFeatureGates: conf.KubeFeatureGates, - Verbosity: verbosity, + Verbosity: env.verbosity, DisableQPSLimits: conf.DisableQPSLimits, ExtraArgs: kubeSchedulerComponentPatches.ExtraArgs, ExtraVolumes: kubeSchedulerComponentPatches.ExtraVolumes, @@ -465,8 +572,13 @@ func (c *Cluster) Install(ctx context.Context) error { if err != nil { return err } - config.Components = append(config.Components, kubeSchedulerComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kubeSchedulerComponent) } + return nil +} + +func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options // Configure the kwok-controller kwokControllerVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.KwokControllerImage, "kwok") @@ -474,7 +586,7 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - kwokControllerComponentPatches := runtime.GetComponentPatches(config, "kwok-controller") + kwokControllerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKwokController) kwokControllerComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(kwokControllerComponentPatches.ExtraVolumes) if err != nil { return fmt.Errorf("failed to expand host volumes for kwok controller component: %w", err) @@ -485,32 +597,37 @@ func (c *Cluster) Install(ctx context.Context) error { kwokControllerExtraVolumes = append(kwokControllerExtraVolumes, logVolumes...) kwokControllerComponent := components.BuildKwokControllerComponent(components.BuildKwokControllerComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Image: conf.KwokControllerImage, Version: kwokControllerVersion, BindAddress: net.PublicAddress, Port: conf.KwokControllerPort, - ConfigPath: kwokConfigPath, - KubeconfigPath: inClusterOnHostKubeconfigPath, - CaCertPath: caCertPath, - AdminCertPath: adminCertPath, - AdminKeyPath: adminKeyPath, + ConfigPath: env.kwokConfigPath, + KubeconfigPath: env.inClusterOnHostKubeconfigPath, + CaCertPath: env.caCertPath, + AdminCertPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, NodeName: c.Name() + "-kwok-controller", - Verbosity: verbosity, + Verbosity: env.verbosity, NodeLeaseDurationSeconds: conf.NodeLeaseDurationSeconds, ExtraArgs: kwokControllerComponentPatches.ExtraArgs, ExtraVolumes: kwokControllerExtraVolumes, ExtraEnvs: kwokControllerComponentPatches.ExtraEnvs, }) - config.Components = append(config.Components, kwokControllerComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kwokControllerComponent) + return nil +} + +func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options // Configure the prometheus if conf.PrometheusPort != 0 { prometheusData, err := BuildPrometheus(BuildPrometheusConfig{ ProjectName: c.Name(), SecurePort: conf.SecurePort, - AdminCrtPath: inClusterAdminCertPath, - AdminKeyPath: inClusterAdminKeyPath, + AdminCrtPath: env.inClusterAdminCertPath, + AdminKeyPath: env.inClusterAdminKeyPath, }) if err != nil { return fmt.Errorf("failed to generate prometheus yaml: %w", err) @@ -529,21 +646,21 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - prometheusComponentPatches := runtime.GetComponentPatches(config, "prometheus") + prometheusComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentPrometheus) prometheusComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(prometheusComponentPatches.ExtraVolumes) if err != nil { return fmt.Errorf("failed to expand host volumes for prometheus component: %w", err) } prometheusComponent, err := components.BuildPrometheusComponent(components.BuildPrometheusComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Image: conf.PrometheusImage, Version: prometheusVersion, BindAddress: net.PublicAddress, Port: conf.PrometheusPort, ConfigPath: prometheusConfigPath, - AdminCertPath: adminCertPath, - AdminKeyPath: adminKeyPath, - Verbosity: verbosity, + AdminCertPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, + Verbosity: env.verbosity, ExtraArgs: prometheusComponentPatches.ExtraArgs, ExtraVolumes: prometheusComponentPatches.ExtraVolumes, ExtraEnvs: prometheusComponentPatches.ExtraEnvs, @@ -551,31 +668,41 @@ func (c *Cluster) Install(ctx context.Context) error { if err != nil { return err } - config.Components = append(config.Components, prometheusComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, prometheusComponent) } + return nil +} + +func (c *Cluster) addDashboard(_ context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options if conf.DashboardPort != 0 { - dashboardComponentPatches := runtime.GetComponentPatches(config, "dashboard") + dashboardComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentDashboard) dashboardComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(dashboardComponentPatches.ExtraVolumes) if err != nil { return fmt.Errorf("failed to expand host volumes for dashboard component: %w", err) } dashboardComponent, err := components.BuildDashboardComponent(components.BuildDashboardComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Image: conf.DashboardImage, BindAddress: net.PublicAddress, - KubeconfigPath: inClusterOnHostKubeconfigPath, - CaCertPath: caCertPath, - AdminCertPath: adminCertPath, - AdminKeyPath: adminKeyPath, + KubeconfigPath: env.inClusterOnHostKubeconfigPath, + CaCertPath: env.caCertPath, + AdminCertPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, Port: conf.DashboardPort, Banner: fmt.Sprintf("Welcome to %s", c.Name()), }) if err != nil { return err } - config.Components = append(config.Components, dashboardComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, dashboardComponent) } + return nil +} + +func (c *Cluster) addJaeger(ctx context.Context, env *env) error { + conf := &env.kwokctlConfig.Options // Configure the jaeger if conf.JaegerPort != 0 { @@ -584,35 +711,40 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - jaegerComponentPatches := runtime.GetComponentPatches(config, "jaeger") + jaegerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentJaeger) jaegerComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(jaegerComponentPatches.ExtraVolumes) if err != nil { return fmt.Errorf("failed to expand host volumes for jaeger component: %w", err) } jaegerComponent, err := components.BuildJaegerComponent(components.BuildJaegerComponentConfig{ - Workdir: workdir, + Workdir: env.workdir, Image: conf.JaegerImage, Version: jaegerVersion, BindAddress: net.PublicAddress, Port: conf.JaegerPort, - Verbosity: verbosity, + Verbosity: env.verbosity, ExtraArgs: jaegerComponentPatches.ExtraArgs, ExtraVolumes: jaegerComponentPatches.ExtraVolumes, }) if err != nil { return err } - config.Components = append(config.Components, jaegerComponent) + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, jaegerComponent) } + return nil +} + +func (c *Cluster) finishInstall(ctx context.Context, env *env) error { + conf := &env.kwokctlConfig.Options // Setup kubeconfig kubeconfigData, err := kubeconfig.EncodeKubeconfig(kubeconfig.BuildKubeconfig(kubeconfig.BuildKubeconfigConfig{ ProjectName: c.Name(), SecurePort: conf.SecurePort, - Address: scheme + "://" + net.LocalAddress + ":" + format.String(conf.KubeApiserverPort), - CACrtPath: caCertPath, - AdminCrtPath: adminCertPath, - AdminKeyPath: adminKeyPath, + Address: env.scheme + "://" + net.LocalAddress + ":" + format.String(conf.KubeApiserverPort), + CACrtPath: env.caCertPath, + AdminCrtPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, })) if err != nil { return err @@ -621,10 +753,10 @@ func (c *Cluster) Install(ctx context.Context) error { inClusterKubeconfigData, err := kubeconfig.EncodeKubeconfig(kubeconfig.BuildKubeconfig(kubeconfig.BuildKubeconfigConfig{ ProjectName: c.Name(), SecurePort: conf.SecurePort, - Address: scheme + "://" + c.Name() + "-kube-apiserver:" + format.String(inClusterPort), - CACrtPath: inClusterCaCertPath, - AdminCrtPath: inClusterAdminCertPath, - AdminKeyPath: inClusterAdminKeyPath, + Address: env.scheme + "://" + c.Name() + "-kube-apiserver:" + format.String(env.inClusterPort), + CACrtPath: env.inClusterCaCertPath, + AdminCrtPath: env.inClusterAdminCertPath, + AdminKeyPath: env.inClusterAdminKeyPath, })) if err != nil { return err @@ -633,7 +765,7 @@ func (c *Cluster) Install(ctx context.Context) error { isSelfCompose := c.isSelfCompose(ctx, true) if !isSelfCompose { composePath := c.GetWorkdirPath(runtime.ComposeName) - compose := convertToCompose(c.Name(), conf.BindAddress, config.Components) + compose := convertToCompose(c.Name(), conf.BindAddress, env.kwokctlConfig.Components) composeData, err := yaml.Marshal(compose) if err != nil { return err @@ -645,17 +777,17 @@ func (c *Cluster) Install(ctx context.Context) error { } // Save config - err = c.WriteFile(kubeconfigPath, kubeconfigData) + err = c.WriteFile(env.kubeconfigPath, kubeconfigData) if err != nil { return err } - err = c.WriteFile(inClusterOnHostKubeconfigPath, inClusterKubeconfigData) + err = c.WriteFile(env.inClusterOnHostKubeconfigPath, inClusterKubeconfigData) if err != nil { return err } - err = c.SetConfig(ctx, config) + err = c.SetConfig(ctx, env.kwokctlConfig) if err != nil { return err } diff --git a/pkg/kwokctl/runtime/compose/cluster_snapshot.go b/pkg/kwokctl/runtime/compose/cluster_snapshot.go index 50af1844c..308fbc7ff 100644 --- a/pkg/kwokctl/runtime/compose/cluster_snapshot.go +++ b/pkg/kwokctl/runtime/compose/cluster_snapshot.go @@ -72,12 +72,12 @@ func (c *Cluster) SnapshotRestore(ctx context.Context, path string) error { etcdContainerName := c.Name() + "-etcd" if conf.Runtime != consts.RuntimeTypeNerdctl { // Restart etcd container - err = c.StopComponent(ctx, "etcd") + err = c.StopComponent(ctx, consts.ComponentEtcd) if err != nil { logger.Error("Failed to stop etcd", err) } defer func() { - err = c.StartComponent(ctx, "etcd") + err = c.StartComponent(ctx, consts.ComponentEtcd) if err != nil { logger.Error("Failed to start etcd", err) } @@ -93,12 +93,12 @@ func (c *Cluster) SnapshotRestore(ctx context.Context, path string) error { // https://github.com/containerd/nerdctl/issues/1812 // Stop the kube-apiserver container to avoid data modification by etcd during restore. - err = c.StopComponent(ctx, "kube-apiserver") + err = c.StopComponent(ctx, consts.ComponentKubeApiserver) if err != nil { logger.Error("Failed to stop kube-apiserver", err) } defer func() { - err = c.StartComponent(ctx, "kube-apiserver") + err = c.StartComponent(ctx, consts.ComponentKubeApiserver) if err != nil { logger.Error("Failed to start kube-apiserver", err) } @@ -111,12 +111,12 @@ func (c *Cluster) SnapshotRestore(ctx context.Context, path string) error { } // Restart etcd container - err = c.StopComponent(ctx, "etcd") + err = c.StopComponent(ctx, consts.ComponentEtcd) if err != nil { logger.Error("Failed to stop etcd", err) } defer func() { - err = c.StartComponent(ctx, "etcd") + err = c.StartComponent(ctx, consts.ComponentEtcd) if err != nil { logger.Error("Failed to start etcd", err) } @@ -138,12 +138,12 @@ func (c *Cluster) SnapshotSaveWithYAML(ctx context.Context, path string, filters // SnapshotRestoreWithYAML restore the snapshot of cluster func (c *Cluster) SnapshotRestoreWithYAML(ctx context.Context, path string, filters []string) error { logger := log.FromContext(ctx) - err := c.StopComponent(ctx, "kube-controller-manager") + err := c.StopComponent(ctx, consts.ComponentKubeControllerManager) if err != nil { logger.Error("Failed to stop kube-controller-manager", err) } defer func() { - err = c.StartComponent(ctx, "kube-controller-manager") + err = c.StartComponent(ctx, consts.ComponentKubeControllerManager) if err != nil { logger.Error("Failed to start kube-controller-manager", err) } diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index 82091b0f9..2475b2127 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -81,6 +81,40 @@ func (c *Cluster) withProviderEnv(ctx context.Context) context.Context { return ctx } +type env struct { + kwokctlConfig *internalversion.KwokctlConfiguration + verbosity log.Level + inClusterKubeconfig string + auditLogPath string + auditPolicyPath string +} + +func (c *Cluster) env(ctx context.Context) (*env, error) { + config, err := c.Config(ctx) + if err != nil { + return nil, err + } + + inClusterKubeconfig := "/etc/kubernetes/scheduler.conf" + auditLogPath := "" + auditPolicyPath := "" + if config.Options.KubeAuditPolicy != "" { + auditLogPath = c.GetLogPath(runtime.AuditLogName) + auditPolicyPath = c.GetWorkdirPath(runtime.AuditPolicyName) + } + + logger := log.FromContext(ctx) + verbosity := logger.Level() + + return &env{ + kwokctlConfig: config, + verbosity: verbosity, + inClusterKubeconfig: inClusterKubeconfig, + auditLogPath: auditLogPath, + auditPolicyPath: auditPolicyPath, + }, nil +} + // Install installs the cluster func (c *Cluster) Install(ctx context.Context) error { err := c.Cluster.Install(ctx) @@ -88,16 +122,42 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - logger := log.FromContext(ctx) - verbosity := logger.Level() - config, err := c.Config(ctx) + env, err := c.env(ctx) if err != nil { return err } - conf := &config.Options - inClusterKubeconfig := "/etc/kubernetes/scheduler.conf" + err = c.addKind(ctx, env) + if err != nil { + return err + } + + err = c.addDashboard(ctx, env) + if err != nil { + return err + } + + err = c.addPrometheus(ctx, env) + if err != nil { + return err + } + err = c.addJaeger(ctx, env) + if err != nil { + return err + } + + err = c.pullAllImages(ctx, env) + if err != nil { + return err + } + + return nil +} + +func (c *Cluster) addKind(ctx context.Context, env *env) (err error) { + logger := log.FromContext(ctx) + conf := &env.kwokctlConfig.Options var featureGates []string var runtimeConfig []string if conf.KubeFeatureGates != "" { @@ -107,22 +167,18 @@ func (c *Cluster) Install(ctx context.Context) error { runtimeConfig = strings.Split(strings.ReplaceAll(conf.KubeRuntimeConfig, "=", ": "), ",") } - auditLogPath := "" - auditPolicyPath := "" if conf.KubeAuditPolicy != "" { err = c.MkdirAll(c.GetWorkdirPath("logs")) if err != nil { return err } - auditLogPath = c.GetLogPath(runtime.AuditLogName) - err = c.CreateFile(auditLogPath) + err = c.CreateFile(env.auditLogPath) if err != nil { return err } - auditPolicyPath = c.GetWorkdirPath(runtime.AuditPolicyName) - err = c.CopyFile(conf.KubeAuditPolicy, auditPolicyPath) + err = c.CopyFile(conf.KubeAuditPolicy, env.auditPolicyPath) if err != nil { return err } @@ -131,7 +187,7 @@ func (c *Cluster) Install(ctx context.Context) error { schedulerConfigPath := "" if !conf.DisableKubeScheduler && conf.KubeSchedulerConfig != "" { schedulerConfigPath = c.GetWorkdirPath(runtime.SchedulerConfigName) - err = c.CopySchedulerConfig(conf.KubeSchedulerConfig, schedulerConfigPath, inClusterKubeconfig) + err = c.CopySchedulerConfig(conf.KubeSchedulerConfig, schedulerConfigPath, env.inClusterKubeconfig) if err != nil { return err } @@ -160,11 +216,11 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - etcdComponentPatches := runtime.GetComponentPatches(config, "etcd") - kubeApiserverComponentPatches := runtime.GetComponentPatches(config, "kube-apiserver") - kubeSchedulerComponentPatches := runtime.GetComponentPatches(config, "kube-scheduler") - kubeControllerManagerComponentPatches := runtime.GetComponentPatches(config, "kube-controller-manager") - kwokControllerComponentPatches := runtime.GetComponentPatches(config, "kwok-controller") + etcdComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentEtcd) + kubeApiserverComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeApiserver) + kubeSchedulerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeScheduler) + kubeControllerManagerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeControllerManager) + kwokControllerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKwokController) extraLogVolumes := runtime.GetLogVolumes(ctx) kwokControllerExtraVolumes := kwokControllerComponentPatches.ExtraVolumes kwokControllerExtraVolumes = append(kwokControllerExtraVolumes, extraLogVolumes...) @@ -184,12 +240,12 @@ func (c *Cluster) Install(ctx context.Context) error { KwokControllerPort: conf.KwokControllerPort, FeatureGates: featureGates, RuntimeConfig: runtimeConfig, - AuditPolicy: auditPolicyPath, - AuditLog: auditLogPath, + AuditPolicy: env.auditPolicyPath, + AuditLog: env.auditLogPath, SchedulerConfig: schedulerConfigPath, ConfigPath: configPath, TracingConfigPath: kubeApiserverTracingConfigPath, - Verbosity: verbosity, + Verbosity: env.verbosity, EtcdExtraArgs: etcdComponentPatches.ExtraArgs, EtcdExtraVolumes: etcdComponentPatches.ExtraVolumes, ApiserverExtraArgs: kubeApiserverComponentPatches.ExtraArgs, @@ -214,7 +270,7 @@ func (c *Cluster) Install(ctx context.Context) error { kwokControllerPod, err := BuildKwokControllerPod(BuildKwokControllerPodConfig{ KwokControllerImage: conf.KwokControllerImage, Name: c.Name(), - Verbosity: verbosity, + Verbosity: env.verbosity, NodeLeaseDurationSeconds: 40, ExtraArgs: kwokControllerComponentPatches.ExtraArgs, ExtraVolumes: kwokControllerExtraVolumes, @@ -227,9 +283,14 @@ func (c *Cluster) Install(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to write %s: %w", runtime.KwokPod, err) } + return nil +} + +func (c *Cluster) addDashboard(_ context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options if conf.DashboardPort != 0 { - dashboardPatches := runtime.GetComponentPatches(config, "dashboard") + dashboardPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentDashboard) dashboardConf := BuildDashboardDeploymentConfig{ DashboardImage: conf.DashboardImage, Name: c.Name(), @@ -247,9 +308,14 @@ func (c *Cluster) Install(ctx context.Context) error { return fmt.Errorf("failed to write %s: %w", runtime.DashboardDeploy, err) } } + return nil +} + +func (c *Cluster) addPrometheus(_ context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options if conf.PrometheusPort != 0 { - prometheusPatches := runtime.GetComponentPatches(config, "prometheus") + prometheusPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentPrometheus) prometheusConf := BuildPrometheusDeploymentConfig{ PrometheusImage: conf.PrometheusImage, Name: c.Name(), @@ -257,8 +323,8 @@ func (c *Cluster) Install(ctx context.Context) error { ExtraVolumes: prometheusPatches.ExtraVolumes, ExtraEnvs: prometheusPatches.ExtraEnvs, } - if verbosity != log.LevelInfo { - prometheusConf.LogLevel = log.ToLogSeverityLevel(verbosity) + if env.verbosity != log.LevelInfo { + prometheusConf.LogLevel = log.ToLogSeverityLevel(env.verbosity) } prometheusDeploy, err := BuildPrometheusDeployment(prometheusConf) if err != nil { @@ -269,9 +335,14 @@ func (c *Cluster) Install(ctx context.Context) error { return fmt.Errorf("failed to write %s: %w", runtime.PrometheusDeploy, err) } } + return nil +} + +func (c *Cluster) addJaeger(_ context.Context, env *env) error { + conf := &env.kwokctlConfig.Options if conf.JaegerPort != 0 { - jaegerPatches := runtime.GetComponentPatches(config, "jaeger") + jaegerPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentJaeger) jaegerConf := BuildJaegerDeploymentConfig{ JaegerImage: conf.JaegerImage, Name: c.Name(), @@ -279,8 +350,8 @@ func (c *Cluster) Install(ctx context.Context) error { ExtraVolumes: jaegerPatches.ExtraVolumes, ExtraEnvs: jaegerPatches.ExtraEnvs, } - if verbosity != log.LevelInfo { - jaegerConf.LogLevel = log.ToLogSeverityLevel(verbosity) + if env.verbosity != log.LevelInfo { + jaegerConf.LogLevel = log.ToLogSeverityLevel(env.verbosity) } jaegerDeploy, err := BuildJaegerDeployment(jaegerConf) if err != nil { @@ -291,12 +362,6 @@ func (c *Cluster) Install(ctx context.Context) error { return fmt.Errorf("failed to write %s: %w", runtime.JaegerDeploy, err) } } - - err = c.pullAllImages(ctx) - if err != nil { - return err - } - return nil } @@ -310,20 +375,20 @@ func (c *Cluster) Up(ctx context.Context) error { config.Components = append(config.Components, internalversion.Component{ - Name: "etcd", + Name: consts.ComponentEtcd, }, internalversion.Component{ - Name: "kube-apiserver", + Name: consts.ComponentKubeApiserver, }, internalversion.Component{ - Name: "kwok-controller", + Name: consts.ComponentKwokController, }, ) if conf.DashboardPort != 0 { config.Components = append(config.Components, internalversion.Component{ - Name: "dashboard", + Name: consts.ComponentDashboard, }, ) } @@ -331,7 +396,7 @@ func (c *Cluster) Up(ctx context.Context) error { if conf.PrometheusPort != 0 { config.Components = append(config.Components, internalversion.Component{ - Name: "prometheus", + Name: consts.ComponentPrometheus, }, ) } @@ -339,7 +404,7 @@ func (c *Cluster) Up(ctx context.Context) error { if conf.JaegerPort != 0 { config.Components = append(config.Components, internalversion.Component{ - Name: "jaeger", + Name: consts.ComponentJaeger, }, ) } @@ -347,7 +412,7 @@ func (c *Cluster) Up(ctx context.Context) error { if !conf.DisableKubeScheduler { config.Components = append(config.Components, internalversion.Component{ - Name: "kube-scheduler", + Name: consts.ComponentKubeScheduler, }, ) } @@ -355,7 +420,7 @@ func (c *Cluster) Up(ctx context.Context) error { if !conf.DisableKubeControllerManager { config.Components = append(config.Components, internalversion.Component{ - Name: "kube-controller-manager", + Name: consts.ComponentKubeControllerManager, }, ) } @@ -473,14 +538,14 @@ func (c *Cluster) Up(ctx context.Context) error { } if conf.DisableKubeScheduler { - err := c.StopComponent(ctx, "kube-scheduler") + err := c.StopComponent(ctx, consts.ComponentKubeScheduler) if err != nil { logger.Error("Failed to disable kube-scheduler", err) } } if conf.DisableKubeControllerManager { - err := c.StopComponent(ctx, "kube-controller-manager") + err := c.StopComponent(ctx, consts.ComponentKubeControllerManager) if err != nil { logger.Error("Failed to disable kube-controller-manager", err) } @@ -489,12 +554,8 @@ func (c *Cluster) Up(ctx context.Context) error { return nil } -func (c *Cluster) pullAllImages(ctx context.Context) error { - config, err := c.Config(ctx) - if err != nil { - return err - } - conf := &config.Options +func (c *Cluster) pullAllImages(ctx context.Context, env *env) error { + conf := &env.kwokctlConfig.Options images := []string{ conf.KindNodeImage, conf.KwokControllerImage, @@ -508,7 +569,7 @@ func (c *Cluster) pullAllImages(ctx context.Context) error { if conf.JaegerPort != 0 { images = append(images, conf.JaegerImage) } - err = c.PullImages(ctx, c.runtime, images, conf.QuietPull) + err := c.PullImages(ctx, c.runtime, images, conf.QuietPull) if err != nil { return err } @@ -707,8 +768,8 @@ func (c *Cluster) Stop(ctx context.Context) error { } var importantComponents = map[string]struct{}{ - "etcd": {}, - "kube-apiserver": {}, + consts.ComponentEtcd: {}, + consts.ComponentKubeApiserver: {}, } // StartComponent starts a component in the cluster @@ -841,7 +902,7 @@ func (c *Cluster) getClusterName() string { func (c *Cluster) getComponentName(name string) string { clusterName := c.getClusterName() switch name { - case "prometheus": + case consts.ComponentPrometheus: default: name = name + "-" + clusterName } @@ -996,7 +1057,7 @@ func (c *Cluster) ListImages(ctx context.Context) ([]string, error) { // EtcdctlInCluster implements the ectdctl subcommand func (c *Cluster) EtcdctlInCluster(ctx context.Context, args ...string) error { - etcdContainerName := c.getComponentName("etcd") + etcdContainerName := c.getComponentName(consts.ComponentEtcd) args = append( []string{ diff --git a/pkg/kwokctl/runtime/kind/cluster_snapshot.go b/pkg/kwokctl/runtime/kind/cluster_snapshot.go index 20af4b3c0..b8a93cbb8 100644 --- a/pkg/kwokctl/runtime/kind/cluster_snapshot.go +++ b/pkg/kwokctl/runtime/kind/cluster_snapshot.go @@ -19,6 +19,7 @@ package kind import ( "context" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/wait" ) @@ -56,19 +57,19 @@ func (c *Cluster) SnapshotSave(ctx context.Context, path string) error { // SnapshotRestore restore the snapshot of cluster func (c *Cluster) SnapshotRestore(ctx context.Context, path string) error { logger := log.FromContext(ctx) - err := c.StopComponent(ctx, "etcd") + err := c.StopComponent(ctx, consts.ComponentEtcd) if err != nil { logger.Error("Failed to stop etcd", err) } defer func() { - err = c.StartComponent(ctx, "etcd") + err = c.StartComponent(ctx, consts.ComponentEtcd) if err != nil { logger.Error("Failed to start etcd", err) } }() // Restore snapshot to host temporary directory - etcdDataTmp := c.GetWorkdirPath("etcd") + etcdDataTmp := c.GetWorkdirPath(consts.ComponentEtcd) err = c.Etcdctl(ctx, "snapshot", "restore", path, "--data-dir", etcdDataTmp) if err != nil { return err @@ -103,14 +104,14 @@ func (c *Cluster) SnapshotSaveWithYAML(ctx context.Context, path string, filters func (c *Cluster) SnapshotRestoreWithYAML(ctx context.Context, path string, filters []string) error { logger := log.FromContext(ctx) err := wait.Poll(ctx, func(ctx context.Context) (bool, error) { - err := c.StopComponent(ctx, "kube-controller-manager") + err := c.StopComponent(ctx, consts.ComponentKubeControllerManager) return err == nil, err }) if err != nil { logger.Error("Failed to stop kube-controller-manager", err) } defer func() { - err = c.StartComponent(ctx, "kube-controller-manager") + err = c.StartComponent(ctx, consts.ComponentKubeControllerManager) if err != nil { logger.Error("Failed to start kube-controller-manager", err) } diff --git a/pkg/kwokctl/snapshot/load.go b/pkg/kwokctl/snapshot/load.go index b364f8ac6..6b6952dca 100644 --- a/pkg/kwokctl/snapshot/load.go +++ b/pkg/kwokctl/snapshot/load.go @@ -103,7 +103,7 @@ func (l *loader) addResource(ctx context.Context, resources []string) { } logger := log.FromContext(ctx) for _, resource := range resources { - mapping, err := mappingFor(l.restMapper, resource) + mapping, err := client.MappingFor(l.restMapper, resource) if err != nil { logger.Warn("Failed to get mapping for resource", "resource", resource, "err", err) continue diff --git a/pkg/kwokctl/snapshot/runtime.go b/pkg/kwokctl/snapshot/runtime.go index e810bf786..3e3a10a2b 100644 --- a/pkg/kwokctl/snapshot/runtime.go +++ b/pkg/kwokctl/snapshot/runtime.go @@ -19,9 +19,7 @@ package snapshot import ( "fmt" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" ) // Resources is the resources of cluster want to save or restore @@ -63,47 +61,5 @@ func clearUnstructured(obj metav1.Object) { obj.SetManagedFields(nil) } -func mappingFor(restMapper meta.RESTMapper, resourceOrKindArg string) (*meta.RESTMapping, error) { - fullySpecifiedGVR, groupResource := schema.ParseResourceArg(resourceOrKindArg) - gvk := schema.GroupVersionKind{} - - if fullySpecifiedGVR != nil { - gvk, _ = restMapper.KindFor(*fullySpecifiedGVR) - } - if gvk.Empty() { - gvk, _ = restMapper.KindFor(groupResource.WithVersion("")) - } - if !gvk.Empty() { - return restMapper.RESTMapping(gvk.GroupKind(), gvk.Version) - } - - fullySpecifiedGVK, groupKind := schema.ParseKindArg(resourceOrKindArg) - if fullySpecifiedGVK == nil { - gvk := groupKind.WithVersion("") - fullySpecifiedGVK = &gvk - } - - if !fullySpecifiedGVK.Empty() { - if mapping, err := restMapper.RESTMapping(fullySpecifiedGVK.GroupKind(), fullySpecifiedGVK.Version); err == nil { - return mapping, nil - } - } - - mapping, err := restMapper.RESTMapping(groupKind, gvk.Version) - if err != nil { - // if we error out here, it is because we could not match a resource or a kind - // for the given argument. To maintain consistency with previous behavior, - // announce that a resource type could not be found. - // if the error is _not_ a *meta.NoKindMatchError, then we had trouble doing discovery, - // so we should return the original error since it may help a user diagnose what is actually wrong - if meta.IsNoMatchError(err) { - return nil, fmt.Errorf("the server doesn't have a resource type %q", groupResource.Resource) - } - return nil, err - } - - return mapping, nil -} - // ErrNotHandled is returned when a resource is not handled var ErrNotHandled = fmt.Errorf("resource not handled") diff --git a/pkg/kwokctl/snapshot/save.go b/pkg/kwokctl/snapshot/save.go index 322893d81..17138a03c 100644 --- a/pkg/kwokctl/snapshot/save.go +++ b/pkg/kwokctl/snapshot/save.go @@ -63,7 +63,7 @@ func Save(ctx context.Context, clientset client.Clientset, w io.Writer, resource gvrs := make([]schema.GroupVersionResource, 0, len(resources)) for _, resource := range resources { - mapping, err := mappingFor(restMapper, resource) + mapping, err := client.MappingFor(restMapper, resource) if err != nil { logger.Warn("Failed to get mapping for resource", "resource", resource, "err", err) continue diff --git a/pkg/utils/client/util.go b/pkg/utils/client/util.go new file mode 100644 index 000000000..38a208d9f --- /dev/null +++ b/pkg/utils/client/util.go @@ -0,0 +1,67 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// MappingFor returns the RESTMapping for the given resource or kind argument. +func MappingFor(restMapper meta.RESTMapper, resourceOrKindArg string) (*meta.RESTMapping, error) { + fullySpecifiedGVR, groupResource := schema.ParseResourceArg(resourceOrKindArg) + gvk := schema.GroupVersionKind{} + + if fullySpecifiedGVR != nil { + gvk, _ = restMapper.KindFor(*fullySpecifiedGVR) + } + if gvk.Empty() { + gvk, _ = restMapper.KindFor(groupResource.WithVersion("")) + } + if !gvk.Empty() { + return restMapper.RESTMapping(gvk.GroupKind(), gvk.Version) + } + + fullySpecifiedGVK, groupKind := schema.ParseKindArg(resourceOrKindArg) + if fullySpecifiedGVK == nil { + gvk := groupKind.WithVersion("") + fullySpecifiedGVK = &gvk + } + + if !fullySpecifiedGVK.Empty() { + if mapping, err := restMapper.RESTMapping(fullySpecifiedGVK.GroupKind(), fullySpecifiedGVK.Version); err == nil { + return mapping, nil + } + } + + mapping, err := restMapper.RESTMapping(groupKind, gvk.Version) + if err != nil { + // if we error out here, it is because we could not match a resource or a kind + // for the given argument. To maintain consistency with previous behavior, + // announce that a resource type could not be found. + // if the error is _not_ a *meta.NoKindMatchError, then we had trouble doing discovery, + // so we should return the original error since it may help a user diagnose what is actually wrong + if meta.IsNoMatchError(err) { + return nil, fmt.Errorf("the server doesn't have a resource type %q", groupResource.Resource) + } + return nil, err + } + + return mapping, nil +}