From 0db20116da74e96b0225676388fb663f2abf964c Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 13 Aug 2024 23:59:47 +0800 Subject: [PATCH] finish operator Signed-off-by: Tom --- .../en/references/fleet_v1alpha1_types.html | 68 +++++++++ examples/fleet/network/submariner-plugin.yaml | 11 +- .../crds/fleet.kurator.dev_fleets.yaml | 21 +++ pkg/apis/fleet/v1alpha1/types.go | 19 ++- pkg/fleet-manager/fleet_plugin_submariner.go | 138 +++++++++++++----- ...mariner-k8s-broker.yaml => sm-broker.yaml} | 2 +- .../manifests/plugins/sm-operator.yaml | 20 +++ .../plugins/submariner-operator.yaml | 13 -- pkg/fleet-manager/plugin/plugin.go | 14 +- pkg/fleet-manager/plugin/plugin_test.go | 6 + .../submariner-k8s-broker/default.yaml | 18 +-- .../testdata/submariner-operator/default.yaml | 34 +++-- 12 files changed, 285 insertions(+), 79 deletions(-) rename pkg/fleet-manager/manifests/plugins/{submariner-k8s-broker.yaml => sm-broker.yaml} (85%) create mode 100644 pkg/fleet-manager/manifests/plugins/sm-operator.yaml delete mode 100644 pkg/fleet-manager/manifests/plugins/submariner-operator.yaml diff --git a/docs/content/en/references/fleet_v1alpha1_types.html b/docs/content/en/references/fleet_v1alpha1_types.html index efd2ba8c..75290cfb 100644 --- a/docs/content/en/references/fleet_v1alpha1_types.html +++ b/docs/content/en/references/fleet_v1alpha1_types.html @@ -333,6 +333,61 @@

BackupStorageLocation +

BrokerConfig +

+

+(Appears on: +SubMarinerConfig) +

+
+
+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+server
+ +string + +		 +

Chart defines the helm chart config of the submariner broker. +default value is +Server is the server address of the broker.

+
+token
+ +string + +		 +

Token is the token for the broker.

+
+ca
+ +string + +		 +

CA is the certificate authority for the broker.

+
+
+

ChartConfig

@@ -1825,6 +1880,19 @@

SubMarinerConfig + + +brokerConfig
+ + +BrokerConfig + + + + +

BrokerConfig defines the configuration for the submariner broker.

+ + diff --git a/examples/fleet/network/submariner-plugin.yaml b/examples/fleet/network/submariner-plugin.yaml index 9662931d..2ac01ddc 100644 --- a/examples/fleet/network/submariner-plugin.yaml +++ b/examples/fleet/network/submariner-plugin.yaml @@ -32,8 +32,11 @@ spec: plugin: submariner: extraArgs: - operator: - image: - pullPolicy: "IfNotPresent" + ipsec: + psk: $SUBMARINER_PSK + broker: + namespace: "submariner" submariner: - natEnabled: false + clusterId: "member1" + clusterCidr: "10.244.0.0/24" + serviceCidr: "10.96.0.0/16" diff --git a/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml b/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml index 209a50d3..4f02631d 100644 --- a/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml +++ b/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml @@ -2786,6 +2786,27 @@ spec: description: SubMariner defines the configuration for the kurator network management. properties: + brokerConfig: + description: BrokerConfig defines the configuration for the + submariner broker. + properties: + ca: + description: CA is the certificate authority for the broker. + type: string + server: + description: |- + Chart defines the helm chart config of the submariner broker. + default value is + Server is the server address of the broker. + type: string + token: + description: Token is the token for the broker. + type: string + required: + - ca + - server + - token + type: object chart: description: |- Chart defines the helm chart config of the submariner. diff --git a/pkg/apis/fleet/v1alpha1/types.go b/pkg/apis/fleet/v1alpha1/types.go index 1a426ffe..4369acda 100644 --- a/pkg/apis/fleet/v1alpha1/types.go +++ b/pkg/apis/fleet/v1alpha1/types.go @@ -597,6 +597,23 @@ type SubMarinerConfig struct { // // +optional ExtraArgs apiextensionsv1.JSON `json:"extraArgs,omitempty"` + // BrokerConfig defines the configuration for the submariner broker. + // +required + BrokerConfig *BrokerConfig `json:"brokerConfig,omitempty"` +} + +type BrokerConfig struct { + // Chart defines the helm chart config of the submariner broker. + // default value is + // Server is the server address of the broker. + // +required + Server string `json:"server"` + // Token is the token for the broker. + // +required + Token string `json:"token"` + // CA is the certificate authority for the broker. + // +required + CA string `json:"ca"` } // Provider only can be istio now. @@ -642,4 +659,4 @@ type FleetList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` Items []Fleet `json:"items"` -} +} \ No newline at end of file diff --git a/pkg/fleet-manager/fleet_plugin_submariner.go b/pkg/fleet-manager/fleet_plugin_submariner.go index a265417b..7d42d5fa 100644 --- a/pkg/fleet-manager/fleet_plugin_submariner.go +++ b/pkg/fleet-manager/fleet_plugin_submariner.go @@ -15,18 +15,68 @@ package fleet import ( "context" + "encoding/base64" "errors" + "fmt" "time" "helm.sh/helm/v3/pkg/kube" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "kurator.dev/kurator/pkg/apis/fleet/v1alpha1" fleetapi "kurator.dev/kurator/pkg/apis/fleet/v1alpha1" "kurator.dev/kurator/pkg/fleet-manager/plugin" "kurator.dev/kurator/pkg/infra/util" ) +var BROKER_NS string = "submariner-k8s-broker" + +func getBrokerInfo(ctx context.Context, key ClusterKey, cluster *FleetCluster) (map[string]string, error) { + defer func() { + if err := recover(); err != nil { + fmt.Println(err) + } + }() + + st_name := fmt.Sprintf("%s-%s-%s-client-token", BROKER_NS, plugin.SubMarinerBrokerComponentName, key.Name) + sts, err := cluster.Client.KubeClient().CoreV1().Secrets(BROKER_NS).Get(ctx, st_name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + broker_ca := base64.StdEncoding.EncodeToString(sts.Data["ca.crt"]) + broker_token := string(sts.Data["token"]) + + endpoints, err := cluster.Client.KubeClient().CoreV1().Endpoints("default").Get(context.TODO(), "kubernetes", metav1.GetOptions{}) + if err != nil { + return nil, err + } + + broker_url := "" + for _, subset := range endpoints.Subsets { + for _, addr := range subset.Addresses { + for _, port := range subset.Ports { + if port.Name == "https" { + broker_url = fmt.Sprintf("%s:%d\n", addr.IP, port.Port) + break + } + } + } + } + if broker_url == "" { + return nil, errors.New("broker url not found") + } + + broker_info := map[string]string{ + "broker_ca": broker_ca, + "broker_token": broker_token, + "broker_url": broker_url, + } + return broker_info, nil +} + // reconcileSubmarinerPlugin reconciles the Submariner plugin. // The fleetClusters parameter is currently unused, but is included to match the function signature of other functions in reconcilePlugins. func (f *FleetManager) reconcileSubmarinerPlugin(ctx context.Context, fleet *fleetapi.Fleet, fleetClusters map[ClusterKey]*FleetCluster) (kube.ResourceList, ctrl.Result, error) { @@ -51,27 +101,72 @@ func (f *FleetManager) reconcileSubmarinerPlugin(ctx context.Context, fleet *fle if len(fleetClusters) < 2 { return nil, ctrl.Result{}, errors.New("fleetClusters number < 2") } + + brokerClusterKey := ClusterKey{ + Kind: fleet.Spec.Clusters[0].Kind, + Name: fleet.Spec.Clusters[0].Name, + } + // Install broker in the first member cluster + log.V(0).Info("broker will be installed in " + brokerClusterKey.Name) + brokerCluster := fleetClusters[brokerClusterKey] + b, err := plugin.RenderSubmarinerBroker(f.Manifests, fleetNN, fleetOwnerRef, plugin.KubeConfigSecretRef{ + Name: brokerClusterKey.Name, + SecretName: brokerCluster.Secret, + SecretKey: brokerCluster.SecretKey, + }, submarinerCfg) + if err != nil { + return nil, ctrl.Result{}, err + } + + brokerResources, err := util.PatchResources(b) + if err != nil { + return nil, ctrl.Result{}, err + } + resources = append(resources, brokerResources...) + + log.V(0).Info("wait for submariner broker helm release to be reconciled") + if !f.helmReleaseReady(ctx, fleet, resources) { + // wait for HelmRelease to be ready + return nil, ctrl.Result{ + // HelmRelease check interval is 1m, so we set 30s here + RequeueAfter: 30 * time.Second, + }, nil + } + + broker_info, err := getBrokerInfo(ctx, brokerClusterKey, brokerCluster) + if err != nil { + log.V(0).Error(err, "failed to get broker info") + return nil, ctrl.Result{}, err + } + + submarinerCfg.BrokerConfig = &v1alpha1.BrokerConfig{ + CA: broker_info["broker_ca"], + Token: broker_info["broker_token"], + Server: broker_info["broker_url"], + } + + // Install operator in all member clusters for key, cluster := range fleetClusters { - b, err := plugin.RenderSubmarinerBroker(f.Manifests, fleetNN, fleetOwnerRef, plugin.KubeConfigSecretRef{ + b, err := plugin.RenderSubmarinerOperator(f.Manifests, fleetNN, fleetOwnerRef, plugin.KubeConfigSecretRef{ Name: key.Name, SecretName: cluster.Secret, SecretKey: cluster.SecretKey, }, submarinerCfg) if err != nil { + log.V(0).Error(err, "failed to render submariner operator") return nil, ctrl.Result{}, err } - brokerResources, err := util.PatchResources(b) + operatorResources, err := util.PatchResources(b) if err != nil { + log.V(0).Error(err, "failed to render submariner operator") return nil, ctrl.Result{}, err } - resources = append(resources, brokerResources...) - log.V(0).Info("broker will be installed in " + key.Name) - // break + resources = append(resources, operatorResources...) } - log.V(0).Info("wait for submariner broker helm release to be reconciled") + log.V(0).Info("wait for submariner operator helm release to be reconciled") if !f.helmReleaseReady(ctx, fleet, resources) { // wait for HelmRelease to be ready return nil, ctrl.Result{ @@ -79,35 +174,6 @@ func (f *FleetManager) reconcileSubmarinerPlugin(ctx context.Context, fleet *fle RequeueAfter: 30 * time.Second, }, nil } - - // Install operator in all member clusters - // for key, cluster := range fleetClusters { - // b, err := plugin.RenderSubmarinerOperator(f.Manifests, fleetNN, fleetOwnerRef, plugin.KubeConfigSecretRef{ - // Name: key.Name, - // SecretName: cluster.Secret, - // SecretKey: cluster.SecretKey, - // }, submarinerCfg) - // if err != nil { - // log.V(0).Error(err, "failed to render submariner operator") - // return nil, ctrl.Result{}, err - // } - - // operatorResources, err := util.PatchResources(b) - // if err != nil { - // log.V(0).Error(err, "failed to render submariner operator") - // return nil, ctrl.Result{}, err - // } - // resources = append(resources, operatorResources...) - // } - - // log.V(0).Info("wait for submariner operator helm release to be reconciled") - // if !f.helmReleaseReady(ctx, fleet, resources) { - // // wait for HelmRelease to be ready - // return nil, ctrl.Result{ - // // HelmRelease check interval is 1m, so we set 30s here - // RequeueAfter: 30 * time.Second, - // }, nil - // } - log.V(0).Info("submariner helm release is ready!!!") + log.V(0).Info("Submariner helm release is ready!") return resources, ctrl.Result{}, nil } diff --git a/pkg/fleet-manager/manifests/plugins/submariner-k8s-broker.yaml b/pkg/fleet-manager/manifests/plugins/sm-broker.yaml similarity index 85% rename from pkg/fleet-manager/manifests/plugins/submariner-k8s-broker.yaml rename to pkg/fleet-manager/manifests/plugins/sm-broker.yaml index be6b7b0a..b7d569bd 100644 --- a/pkg/fleet-manager/manifests/plugins/submariner-k8s-broker.yaml +++ b/pkg/fleet-manager/manifests/plugins/sm-broker.yaml @@ -2,7 +2,7 @@ type: default repo: https://submariner-io.github.io/submariner-charts/charts name: submariner-k8s-broker version: 0.18.0 -targetNamespace: submariner +targetNamespace: submariner-k8s-broker values: rbac: create: true diff --git a/pkg/fleet-manager/manifests/plugins/sm-operator.yaml b/pkg/fleet-manager/manifests/plugins/sm-operator.yaml new file mode 100644 index 00000000..b0240eea --- /dev/null +++ b/pkg/fleet-manager/manifests/plugins/sm-operator.yaml @@ -0,0 +1,20 @@ +type: default +repo: https://submariner-io.github.io/submariner-charts/charts +name: submariner-operator +version: 0.18.0 +targetNamespace: submariner-operator +values: + broker: + globalnet: true + submariner: + serviceDiscovery: true + cableDriver: "libreswan" + globalCidr: "242.0.0.0/8" + natEnabled: false + serviceAccounts: + globalnet: + create: "242.0.0.0/8" + lighthouseAgent: + create: true + lighthouseCoreDns: + create: true diff --git a/pkg/fleet-manager/manifests/plugins/submariner-operator.yaml b/pkg/fleet-manager/manifests/plugins/submariner-operator.yaml deleted file mode 100644 index 3ae82164..00000000 --- a/pkg/fleet-manager/manifests/plugins/submariner-operator.yaml +++ /dev/null @@ -1,13 +0,0 @@ -type: default -repo: https://submariner-io.github.io/submariner-charts/charts -name: submariner-operator -version: 0.18.0 -targetNamespace: submariner -values: - rbac: - create: true - submariner: - natEnabled: false - operator: - image: - tag: "0.14.0" diff --git a/pkg/fleet-manager/plugin/plugin.go b/pkg/fleet-manager/plugin/plugin.go index 302d81db..f055378d 100644 --- a/pkg/fleet-manager/plugin/plugin.go +++ b/pkg/fleet-manager/plugin/plugin.go @@ -40,7 +40,7 @@ const ( ClusterStoragePluginName = "cluster-storage" FlaggerPluginName = "flagger" PublicTestloaderName = "testloader" - SubMarinerBrokerPluginName = "submariner-k8s-broker" + SubMarinerBrokerPluginName = "submariner-broker" SubMarinerOperatorPluginName = "submariner-operator" ThanosComponentName = "thanos" @@ -53,8 +53,8 @@ const ( RookClusterComponentName = "rook-ceph" FlaggerComponentName = "flagger" TestloaderComponentName = "testloader" - SubMarinerBrokerComponentName = "submariner-k8s-broker" - SubMarinerOperatorComponentName = "submariner-operator" + SubMarinerBrokerComponentName = "sm-broker" + SubMarinerOperatorComponentName = "sm-operator" OCIReposiotryPrefix = "oci://" ) @@ -497,6 +497,14 @@ func RenderSubmarinerOperator( return nil, err } + values = transform.MergeMaps(values, map[string]interface{}{ + "broker": map[string]interface{}{ + "server": subMarinerConfig.BrokerConfig.Server, + "token": subMarinerConfig.BrokerConfig.Token, + "ca": subMarinerConfig.BrokerConfig.CA, + }, + }) + return renderFleetPlugin(fsys, FleetPluginConfig{ Name: SubMarinerOperatorPluginName, Component: SubMarinerOperatorComponentName, diff --git a/pkg/fleet-manager/plugin/plugin_test.go b/pkg/fleet-manager/plugin/plugin_test.go index ec6b9c1c..77e5152b 100644 --- a/pkg/fleet-manager/plugin/plugin_test.go +++ b/pkg/fleet-manager/plugin/plugin_test.go @@ -709,6 +709,12 @@ func TestRenderSubmarinerOperator(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { + tc.config.BrokerConfig = &v1alpha1.BrokerConfig{ + Server: "server-xxx", + Token: "token-xxx", + CA: "ca-xxx", + } + got, err := RenderSubmarinerOperator(manifestFS, tc.fleet, tc.ref, KubeConfigSecretRef{ Name: "cluster1", SecretName: "cluster1", diff --git a/pkg/fleet-manager/plugin/testdata/submariner-k8s-broker/default.yaml b/pkg/fleet-manager/plugin/testdata/submariner-k8s-broker/default.yaml index 57c7eee1..949972bd 100644 --- a/pkg/fleet-manager/plugin/testdata/submariner-k8s-broker/default.yaml +++ b/pkg/fleet-manager/plugin/testdata/submariner-k8s-broker/default.yaml @@ -1,13 +1,13 @@ apiVersion: source.toolkit.fluxcd.io/v1beta2 kind: HelmRepository metadata: - name: "submariner-k8s-broker-cluster1" + name: "sm-broker-cluster1" namespace: "default" labels: app.kubernetes.io/managed-by: fleet-manager fleet.kurator.dev/name: "fleet-1" - fleet.kurator.dev/plugin: "submariner-k8s-broker" - fleet.kurator.dev/component: "submariner-k8s-broker" + fleet.kurator.dev/plugin: "submariner-broker" + fleet.kurator.dev/component: "sm-broker" ownerReferences: - apiVersion: "fleet.kurator.dev/v1alpha1" kind: "Fleet" @@ -21,13 +21,13 @@ spec: apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease metadata: - name: "submariner-k8s-broker-cluster1" + name: "sm-broker-cluster1" namespace: "default" labels: app.kubernetes.io/managed-by: fleet-manager fleet.kurator.dev/name: "fleet-1" - fleet.kurator.dev/plugin: "submariner-k8s-broker" - fleet.kurator.dev/component: "submariner-k8s-broker" + fleet.kurator.dev/plugin: "submariner-broker" + fleet.kurator.dev/component: "sm-broker" ownerReferences: - apiVersion: "fleet.kurator.dev/v1alpha1" kind: "Fleet" @@ -40,7 +40,7 @@ spec: version: "0.18.0" sourceRef: kind: HelmRepository - name: "submariner-k8s-broker-cluster1" + name: "sm-broker-cluster1" values: crd: create: true @@ -52,8 +52,8 @@ spec: interval: 1m0s install: createNamespace: true - targetNamespace: "submariner" - storageNamespace: "submariner" + targetNamespace: "submariner-k8s-broker" + storageNamespace: "submariner-k8s-broker" timeout: 15m0s kubeConfig: secretRef: diff --git a/pkg/fleet-manager/plugin/testdata/submariner-operator/default.yaml b/pkg/fleet-manager/plugin/testdata/submariner-operator/default.yaml index 7794c6a8..59e2d357 100644 --- a/pkg/fleet-manager/plugin/testdata/submariner-operator/default.yaml +++ b/pkg/fleet-manager/plugin/testdata/submariner-operator/default.yaml @@ -1,13 +1,13 @@ apiVersion: source.toolkit.fluxcd.io/v1beta2 kind: HelmRepository metadata: - name: "submariner-operator-cluster1" + name: "sm-operator-cluster1" namespace: "default" labels: app.kubernetes.io/managed-by: fleet-manager fleet.kurator.dev/name: "fleet-1" fleet.kurator.dev/plugin: "submariner-operator" - fleet.kurator.dev/component: "submariner-operator" + fleet.kurator.dev/component: "sm-operator" ownerReferences: - apiVersion: "fleet.kurator.dev/v1alpha1" kind: "Fleet" @@ -21,13 +21,13 @@ spec: apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease metadata: - name: "submariner-operator-cluster1" + name: "sm-operator-cluster1" namespace: "default" labels: app.kubernetes.io/managed-by: fleet-manager fleet.kurator.dev/name: "fleet-1" fleet.kurator.dev/plugin: "submariner-operator" - fleet.kurator.dev/component: "submariner-operator" + fleet.kurator.dev/component: "sm-operator" ownerReferences: - apiVersion: "fleet.kurator.dev/v1alpha1" kind: "Fleet" @@ -40,20 +40,30 @@ spec: version: "0.18.0" sourceRef: kind: HelmRepository - name: "submariner-operator-cluster1" + name: "sm-operator-cluster1" values: - operator: - image: - tag: 0.14.0 - rbac: - create: true + broker: + ca: ca-xxx + globalnet: true + server: server-xxx + token: token-xxx + serviceAccounts: + globalnet: + create: 242.0.0.0/8 + lighthouseAgent: + create: true + lighthouseCoreDns: + create: true submariner: + cableDriver: libreswan + globalCidr: 242.0.0.0/8 natEnabled: false + serviceDiscovery: true interval: 1m0s install: createNamespace: true - targetNamespace: "submariner" - storageNamespace: "submariner" + targetNamespace: "submariner-operator" + storageNamespace: "submariner-operator" timeout: 15m0s kubeConfig: secretRef: