Skip to content

Commit

Permalink
finish operator
Browse files Browse the repository at this point in the history 
Signed-off-by: Tom <yusencao@outlook.com>
  • Loading branch information
@Flying-Tom
Flying-Tom committed Aug 13, 2024
1 parent a4b3db4 commit 0db2011
Show file tree
Hide file tree
Showing 12 changed files with 285 additions and 79 deletions.
68 changes: 68 additions & 0 deletions docs/content/en/references/fleet_v1alpha1_types.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,61 @@ <h3 id="fleet.kurator.dev/v1alpha1.BackupStorageLocation">BackupStorageLocation
</table>
</div>
</div>
<h3 id="fleet.kurator.dev/v1alpha1.BrokerConfig">BrokerConfig
</h3>
<p>
(<em>Appears on:</em>
<a href="#fleet.kurator.dev/v1alpha1.SubMarinerConfig">SubMarinerConfig</a>)
</p>
<div class="md-typeset__scrollwrap">
<div class="md-typeset__table td-content">
<table>
<thead>
<tr>
<th>Field</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<code>server</code><br>
<em>
string
</em>
</td>
<td>
<p>Chart defines the helm chart config of the submariner broker.
default value is
Server is the server address of the broker.</p>
</td>
</tr>
<tr>
<td>
<code>token</code><br>
<em>
string
</em>
</td>
<td>
<p>Token is the token for the broker.</p>
</td>
</tr>
<tr>
<td>
<code>ca</code><br>
<em>
string
</em>
</td>
<td>
<p>CA is the certificate authority for the broker.</p>
</td>
</tr>
</tbody>
</table>
</div>
</div>
<h3 id="fleet.kurator.dev/v1alpha1.ChartConfig">ChartConfig
</h3>
<p>
Expand Down Expand Up @@ -1825,6 +1880,19 @@ <h3 id="fleet.kurator.dev/v1alpha1.SubMarinerConfig">SubMarinerConfig
</code></pre>
</td>
</tr>
<tr>
<td>
<code>brokerConfig</code><br>
<em>
<a href="#fleet.kurator.dev/v1alpha1.BrokerConfig">
BrokerConfig
</a>
</em>
</td>
<td>
<p>BrokerConfig defines the configuration for the submariner broker.</p>
</td>
</tr>
</tbody>
</table>
</div>
Expand Down
11 changes: 7 additions & 4 deletions examples/fleet/network/submariner-plugin.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,11 @@ spec:
plugin:
submariner:
extraArgs:
operator:
image:
pullPolicy: "IfNotPresent"
ipsec:
psk: $SUBMARINER_PSK
broker:
namespace: "submariner"
submariner:
natEnabled: false
clusterId: "member1"
clusterCidr: "10.244.0.0/24"
serviceCidr: "10.96.0.0/16"
21 changes: 21 additions & 0 deletions manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2786,6 +2786,27 @@ spec:
description: SubMariner defines the configuration for the kurator
network management.
properties:
brokerConfig:
description: BrokerConfig defines the configuration for the
submariner broker.
properties:
ca:
description: CA is the certificate authority for the broker.
type: string
server:
description: |-
Chart defines the helm chart config of the submariner broker.
default value is
Server is the server address of the broker.
type: string
token:
description: Token is the token for the broker.
type: string
required:
- ca
- server
- token
type: object
chart:
description: |-
Chart defines the helm chart config of the submariner.
Expand Down
19 changes: 18 additions & 1 deletion pkg/apis/fleet/v1alpha1/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -597,6 +597,23 @@ type SubMarinerConfig struct {
//
// +optional
ExtraArgs apiextensionsv1.JSON `json:"extraArgs,omitempty"`
// BrokerConfig defines the configuration for the submariner broker.
// +required
BrokerConfig *BrokerConfig `json:"brokerConfig,omitempty"`
}

type BrokerConfig struct {
// Chart defines the helm chart config of the submariner broker.
// default value is
// Server is the server address of the broker.
// +required
Server string `json:"server"`
// Token is the token for the broker.
// +required
Token string `json:"token"`
// CA is the certificate authority for the broker.
// +required
CA string `json:"ca"`
}

// Provider only can be istio now.
Expand Down Expand Up @@ -642,4 +659,4 @@ type FleetList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []Fleet `json:"items"`
}
}
138 changes: 102 additions & 36 deletions pkg/fleet-manager/fleet_plugin_submariner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,68 @@ package fleet

import (
"context"
"encoding/base64"
"errors"
"fmt"
"time"

"helm.sh/helm/v3/pkg/kube"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"kurator.dev/kurator/pkg/apis/fleet/v1alpha1"
fleetapi "kurator.dev/kurator/pkg/apis/fleet/v1alpha1"
"kurator.dev/kurator/pkg/fleet-manager/plugin"
"kurator.dev/kurator/pkg/infra/util"
)

var BROKER_NS string = "submariner-k8s-broker"

func getBrokerInfo(ctx context.Context, key ClusterKey, cluster *FleetCluster) (map[string]string, error) {
defer func() {
if err := recover(); err != nil {
fmt.Println(err)
}
}()

st_name := fmt.Sprintf("%s-%s-%s-client-token", BROKER_NS, plugin.SubMarinerBrokerComponentName, key.Name)
sts, err := cluster.Client.KubeClient().CoreV1().Secrets(BROKER_NS).Get(ctx, st_name, metav1.GetOptions{})
if err != nil {
return nil, err
}

broker_ca := base64.StdEncoding.EncodeToString(sts.Data["ca.crt"])
broker_token := string(sts.Data["token"])

endpoints, err := cluster.Client.KubeClient().CoreV1().Endpoints("default").Get(context.TODO(), "kubernetes", metav1.GetOptions{})
if err != nil {
return nil, err
}

broker_url := ""
for _, subset := range endpoints.Subsets {
for _, addr := range subset.Addresses {
for _, port := range subset.Ports {
if port.Name == "https" {
broker_url = fmt.Sprintf("%s:%d\n", addr.IP, port.Port)
break
}
}
}
}
if broker_url == "" {
return nil, errors.New("broker url not found")
}

broker_info := map[string]string{
"broker_ca": broker_ca,
"broker_token": broker_token,
"broker_url": broker_url,
}
return broker_info, nil
}

// reconcileSubmarinerPlugin reconciles the Submariner plugin.
// The fleetClusters parameter is currently unused, but is included to match the function signature of other functions in reconcilePlugins.
func (f *FleetManager) reconcileSubmarinerPlugin(ctx context.Context, fleet *fleetapi.Fleet, fleetClusters map[ClusterKey]*FleetCluster) (kube.ResourceList, ctrl.Result, error) {
Expand All @@ -51,63 +101,79 @@ func (f *FleetManager) reconcileSubmarinerPlugin(ctx context.Context, fleet *fle
if len(fleetClusters) < 2 {
return nil, ctrl.Result{}, errors.New("fleetClusters number < 2")
}

brokerClusterKey := ClusterKey{
Kind: fleet.Spec.Clusters[0].Kind,
Name: fleet.Spec.Clusters[0].Name,
}

// Install broker in the first member cluster
log.V(0).Info("broker will be installed in " + brokerClusterKey.Name)
brokerCluster := fleetClusters[brokerClusterKey]
b, err := plugin.RenderSubmarinerBroker(f.Manifests, fleetNN, fleetOwnerRef, plugin.KubeConfigSecretRef{
Name: brokerClusterKey.Name,
SecretName: brokerCluster.Secret,
SecretKey: brokerCluster.SecretKey,
}, submarinerCfg)
if err != nil {
return nil, ctrl.Result{}, err
}

brokerResources, err := util.PatchResources(b)
if err != nil {
return nil, ctrl.Result{}, err
}
resources = append(resources, brokerResources...)

log.V(0).Info("wait for submariner broker helm release to be reconciled")
if !f.helmReleaseReady(ctx, fleet, resources) {
// wait for HelmRelease to be ready
return nil, ctrl.Result{
// HelmRelease check interval is 1m, so we set 30s here
RequeueAfter: 30 * time.Second,
}, nil
}

broker_info, err := getBrokerInfo(ctx, brokerClusterKey, brokerCluster)
if err != nil {
log.V(0).Error(err, "failed to get broker info")
return nil, ctrl.Result{}, err
}

submarinerCfg.BrokerConfig = &v1alpha1.BrokerConfig{
CA: broker_info["broker_ca"],
Token: broker_info["broker_token"],
Server: broker_info["broker_url"],
}

// Install operator in all member clusters
for key, cluster := range fleetClusters {
b, err := plugin.RenderSubmarinerBroker(f.Manifests, fleetNN, fleetOwnerRef, plugin.KubeConfigSecretRef{
b, err := plugin.RenderSubmarinerOperator(f.Manifests, fleetNN, fleetOwnerRef, plugin.KubeConfigSecretRef{
Name: key.Name,
SecretName: cluster.Secret,
SecretKey: cluster.SecretKey,
}, submarinerCfg)
if err != nil {
log.V(0).Error(err, "failed to render submariner operator")
return nil, ctrl.Result{}, err
}

brokerResources, err := util.PatchResources(b)
operatorResources, err := util.PatchResources(b)
if err != nil {
log.V(0).Error(err, "failed to render submariner operator")
return nil, ctrl.Result{}, err
}
resources = append(resources, brokerResources...)
log.V(0).Info("broker will be installed in " + key.Name)
// break
resources = append(resources, operatorResources...)
}

log.V(0).Info("wait for submariner broker helm release to be reconciled")
log.V(0).Info("wait for submariner operator helm release to be reconciled")
if !f.helmReleaseReady(ctx, fleet, resources) {
// wait for HelmRelease to be ready
return nil, ctrl.Result{
// HelmRelease check interval is 1m, so we set 30s here
RequeueAfter: 30 * time.Second,
}, nil
}

// Install operator in all member clusters
// for key, cluster := range fleetClusters {
// b, err := plugin.RenderSubmarinerOperator(f.Manifests, fleetNN, fleetOwnerRef, plugin.KubeConfigSecretRef{
// Name: key.Name,
// SecretName: cluster.Secret,
// SecretKey: cluster.SecretKey,
// }, submarinerCfg)
// if err != nil {
// log.V(0).Error(err, "failed to render submariner operator")
// return nil, ctrl.Result{}, err
// }

// operatorResources, err := util.PatchResources(b)
// if err != nil {
// log.V(0).Error(err, "failed to render submariner operator")
// return nil, ctrl.Result{}, err
// }
// resources = append(resources, operatorResources...)
// }

// log.V(0).Info("wait for submariner operator helm release to be reconciled")
// if !f.helmReleaseReady(ctx, fleet, resources) {
// // wait for HelmRelease to be ready
// return nil, ctrl.Result{
// // HelmRelease check interval is 1m, so we set 30s here
// RequeueAfter: 30 * time.Second,
// }, nil
// }
log.V(0).Info("submariner helm release is ready!!!")
log.V(0).Info("Submariner helm release is ready!")
return resources, ctrl.Result{}, nil
}
2 changes: 1 addition & 1 deletion ...ifests/plugins/submariner-k8s-broker.yaml → ...-manager/manifests/plugins/sm-broker.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ type: default
repo: https://submariner-io.github.io/submariner-charts/charts
name: submariner-k8s-broker
version: 0.18.0
targetNamespace: submariner
targetNamespace: submariner-k8s-broker
values:
rbac:
create: true
Expand Down
20 changes: 20 additions & 0 deletions pkg/fleet-manager/manifests/plugins/sm-operator.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
type: default
repo: https://submariner-io.github.io/submariner-charts/charts
name: submariner-operator
version: 0.18.0
targetNamespace: submariner-operator
values:
broker:
globalnet: true
submariner:
serviceDiscovery: true
cableDriver: "libreswan"
globalCidr: "242.0.0.0/8"
natEnabled: false
serviceAccounts:
globalnet:
create: "242.0.0.0/8"
lighthouseAgent:
create: true
lighthouseCoreDns:
create: true
13 changes: 0 additions & 13 deletions pkg/fleet-manager/manifests/plugins/submariner-operator.yaml
View file

This file was deleted.

Loading

0 comments on commit 0db2011

Please sign in to comment.