From ac9a94e63f9fddcf5c0365e7f7f4f5c45e5799cf Mon Sep 17 00:00:00 2001
From: Britania Rodriguez Reyes <145056127+britaniar@users.noreply.github.com>
Date: Fri, 5 Apr 2024 19:44:36 -0700
Subject: [PATCH] fleet setup scripts (#716)

---
 hack/Azure/setup/README.md           | 117 +++++++++++++++++++++++++++
 hack/Azure/setup/createHubCluster.sh |  71 ++++++++++++++++
 hack/Azure/setup/joinMC.sh           |  94 +++++++++++++++++++++
 3 files changed, 282 insertions(+)
 create mode 100644 hack/Azure/setup/README.md
 create mode 100644 hack/Azure/setup/createHubCluster.sh
 create mode 100644 hack/Azure/setup/joinMC.sh

diff --git a/hack/Azure/setup/README.md b/hack/Azure/setup/README.md
new file mode 100644
index 000000000..447d4009e
--- /dev/null
+++ b/hack/Azure/setup/README.md
@@ -0,0 +1,117 @@
+# Setting up a Fleet
+
+This how-to guide describes how to create a fleet using Azure Kubernetes Service, specifically:
+
+* how to create an AKS cluster as the  hub cluster; and
+* how to join any k8s clusters to the AKS hub cluster; and
+* how to add labels to a member cluster representation on the hub cluster
+
+> Note
+>
+> To run these scripts, make sure that you have already installed the following tools in your
+> system:
+> * `kubectl`, the Kubernetes CLI
+> * `helm`, a Kubernetes package manager
+> * `curl`
+> * `jq`
+> * `base64`
+> 
+>  
+> Also, make sure that you have already cloned the repo and are in the root directory.
+> * `git clone https://github.com/Azure/fleet.git`
+> * `cd fleet`
+
+## Create a hub cluster from an AKS Cluster
+For your convenience, Fleet provides a script that can automate the process of creating a hub cluster. To use script,
+run the commands bellow:
+```sh
+# Replace the value of <AZURE-SUBSCRIPTION-ID> with your Azure subscription ID.
+export SUB=<AZURE-SUBSCRIPTION-ID>
+
+# Run the script. Be sure to replace the values of <RESOURCE-GROUP-NAME>, <LOCATION>, and <HUB-CLUSTER-NAME> with those of your own.
+chmod +x hack/Azure/setup/createHubCluster.sh
+./hack/Azure/setup/createHubCluster.sh <RESOURCE-GROUP-NAME> <LOCATION> <HUB-CLUSTER-NAME>
+```
+
+It may take a few minutes for the script to finish running. Once it is completed, verify that the `hub-agent` has been installed:
+```
+kubectl get pods -n fleet-system
+```
+
+If you would like to add a prometheus server to access metrics, run the following: 
+<details>
+<summary> Add Prometheus Server</summary>
+
+1.  Check the status of the service. Copy the `EXTERNAL-IP` of the `fleet-prometheus-endpoint` from the services for later.
+    ````
+    kubectl get service -n fleet-system
+    ````
+
+2. Install the Prometheus community Helm Chart
+   ```
+    helm install prom prometheus-community/kube-prometheus-stack -f prom1.yaml
+   ```
+   The `prom1.yaml` file should contain the following YAML code:
+    ```yaml
+    prometheus:
+        service:
+            type: LoadBalancer
+        prometheusSpec:
+            additionalScrapeConfigs:
+            - job_name: "fleet"
+              static_configs:
+              - targets: [<EXTERNAL-IP>:8080"]
+    ```
+    Replace `<EXTERNAL-IP>` with the external IP address obtained previously.
+</details>
+
+
+## Joining a cluster onto hub cluster
+
+A cluster can join in a hub cluster if:
+
+* it runs a supported Kubernetes version; it is recommended that you use Kubernetes 1.28 or later
+  versions, and
+* it has network connectivity to the hub cluster.
+
+> Note
+>
+> To run this script, make sure you have already created cluster(s) and gotten their credentials.
+> 
+
+For your convenience, Fleet provides a script that can automate the process of joining a cluster
+onto a hub cluster. To use the script, run the commands below after creating needed AKS clusters:
+```sh
+# Pass in the hub cluster name and a list of cluster names (separated by a space) as arguments to the script that you would like to 
+# join the fleet as member clusters. Their context will be used to access the cluster.
+# Ex.: ./hack/setup/joinMC.sh test-hub member member2 member3 
+# Run the script.
+chmod +x hack/Azure/setup/joinMC.sh
+./hack/Azure/setup/joinMC.sh <HUB-CLUSTER-NAME> <MEMBER-CLUSTER-NAME-1> <MEMBER-CLUSTER-NAME-2> <MEMBER-CLUSTER-NAME-3> <MEMBER-CLUSTER-NAME-4>
+```
+
+It may take a few minutes for the script to finish running. Once it is completed, verify
+that the cluster has joined successfully with the command below:
+
+```sh
+kubectl get membercluster $MEMBER_CLUSTER
+```
+
+If you see that the cluster is still in an unknown state, it might be that the member cluster
+is still connecting to the hub cluster. Should this state persist for a prolonged
+period, refer to the [Troubleshooting Guide](../../../docs/troubleshooting/README.md) for
+more information.
+
+## Adding labels to a member cluster
+
+You can add labels to a `MemberCluster` object in the same as with any other Kubernetes object.
+These labels can then be used for targeting specific clusters in resource placement. To add a label,
+run the command below:
+
+```sh
+# Replace the values of MEMBER_CLUSTER, LABEL_KEY, and LABEL_VALUE with those of your own.
+export MEMBER_CLUSTER=YOUR-MEMBER-CLUSTER
+export LABEL_KEY=YOUR-LABEL-KEY
+export LABEL_VALUE=YOUR-LABEL-VALUE
+kubectl label membercluster $MEMBER_CLUSTER $LABEL_KEY=$LABEL_VALUE
+```
\ No newline at end of file
diff --git a/hack/Azure/setup/createHubCluster.sh b/hack/Azure/setup/createHubCluster.sh
new file mode 100644
index 000000000..751e80ff0
--- /dev/null
+++ b/hack/Azure/setup/createHubCluster.sh
@@ -0,0 +1,71 @@
+# This script creates a Hub CLuster from an AKS Cluster (AKS Cluster and Container Registry must be created beforehand).
+
+export RESOURCE_GROUP=$1
+export LOCATION=$2
+export HUB_CLUSTER=$3
+
+az account set -s ${SUB}
+az group create --name $RESOURCE_GROUP --location $LOCATION
+az aks create --resource-group $RESOURCE_GROUP --name $HUB_CLUSTER --node-count 2
+az aks get-credentials --resource-group $RESOURCE_GROUP --name $HUB_CLUSTER
+
+export HUB_CLUSTER_CONTEXT=$(kubectl config view -o jsonpath="{.contexts[?(@.context.cluster==\"$HUB_CLUSTER\")].name}")
+export HUB_CLUSTER_ADDRESS=$(kubectl config view -o jsonpath="{.clusters[?(@.name==\"$HUB_CLUSTER\")].cluster.server}")
+
+
+kubectl config use-context $HUB_CLUSTER_CONTEXT
+
+# Retrieve the hub agent image
+echo "Retrieving hub-agent image..."
+export REGISTRY="mcr.microsoft.com/aks/fleet"
+export TAG=$(curl "https://api.github.com/repos/Azure/fleet/tags" | jq -r '.[0].name')
+export OUTPUT_TYPE="${OUTPUT_TYPE:-type=docker}"
+
+
+echo "Installing hub-agent..."
+# Install the hub agent helm chart on the hub cluster
+helm install hub-agent charts/hub-agent/ \
+  --set image.pullPolicy=Always \
+  --set image.repository=$REGISTRY/hub-agent \
+  --set image.tag=$TAG \
+  --set logVerbosity=2 \
+  --set namespace=fleet-system \
+  --set enableWebhook=false \
+  --set webhookClientConnectionType=service \
+  --set enableV1Alpha1APIs=false \
+  --set enableV1Beta1APIs=true \
+  --set resources.limits.cpu=4 \
+  --set resources.limits.memory=4Gi \
+  --set concurrentClusterPlacementSyncs=10 \
+  --set ConcurrentRolloutSyncs=20 \
+  --set hubAPIQPS=100 \
+  --set hubAPIBurst=1000 \
+  --set logFileMaxSize=100000000 \
+  --set MaxFleetSizeSupported=100
+
+# Check the status of the hub agent
+kubectl get pods -n fleet-system
+
+echo "Installing prometheus endpoint..."
+# Update prometheus and grafana to the hub cluster
+helm repo update
+
+# Install prometheus fleet metrics
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Service
+metadata:
+  name: fleet-prometheus-endpoint
+  namespace: fleet-system
+spec:
+  selector:
+    app.kubernetes.io/name: hub-agent
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080
+  type: LoadBalancer
+EOF
+
+# Check the status of the service
+kubectl get service -n fleet-system
\ No newline at end of file
diff --git a/hack/Azure/setup/joinMC.sh b/hack/Azure/setup/joinMC.sh
new file mode 100644
index 000000000..1947ff0b8
--- /dev/null
+++ b/hack/Azure/setup/joinMC.sh
@@ -0,0 +1,94 @@
+# CAN ONLY BE RUN AFTER CREATING NEEDED AKS CLUSTERS AND HUB CLUSTER. This script creates member clusters from
+# AKS Cluster's and joins them onto the hub cluster.
+
+export HUB_CLUSTER=$1
+export HUB_CLUSTER_CONTEXT=$(kubectl config view -o jsonpath="{.contexts[?(@.context.cluster==\"$HUB_CLUSTER\")].name}")
+export HUB_CLUSTER_ADDRESS=$(kubectl config view -o jsonpath="{.clusters[?(@.name==\"$HUB_CLUSTER\")].cluster.server}")
+
+for MC in "${@:2}"; do
+
+# Note that Fleet will recognize your cluster with this name once it joins.
+export MEMBER_CLUSTER=${MC}
+export MEMBER_CLUSTER_CONTEXT=${MC}
+
+export SERVICE_ACCOUNT="$MEMBER_CLUSTER-hub-cluster-access"
+
+#echo "Switching into hub cluster context..."
+kubectl config use-context $HUB_CLUSTER_CONTEXT
+# The service account can, in theory, be created in any namespace; for simplicity reasons,
+# here you will use the namespace reserved by Fleet installation, `fleet-system`.
+#
+# Note that if you choose a different value, commands in some steps below need to be
+# modified accordingly.
+echo "Creating member service account..."
+kubectl create serviceaccount $SERVICE_ACCOUNT -n fleet-system
+
+echo "Creating member service account secret..."
+export SERVICE_ACCOUNT_SECRET="$MEMBER_CLUSTER-hub-cluster-access-token"
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Secret
+metadata:
+    name: $SERVICE_ACCOUNT_SECRET
+    namespace: fleet-system
+    annotations:
+        kubernetes.io/service-account.name: $SERVICE_ACCOUNT
+type: kubernetes.io/service-account-token
+EOF
+
+echo "Creating member cluster..."
+export TOKEN=$(kubectl get secret $SERVICE_ACCOUNT_SECRET -n fleet-system -o jsonpath='{.data.token}' | base64 -d)
+cat <<EOF | kubectl apply -f -
+apiVersion: cluster.kubernetes-fleet.io/v1beta1
+kind: MemberCluster
+metadata:
+    name: $MEMBER_CLUSTER
+spec:
+    identity:
+        name: $MEMBER_CLUSTER-hub-cluster-access
+        kind: ServiceAccount
+        namespace: fleet-system
+        apiGroup: ""
+    heartbeatPeriodSeconds: 60
+EOF
+
+# # Install the member agent helm chart on the member cluster.
+
+# The variables below uses the Fleet images kept in the Microsoft Container Registry (MCR),
+# and will retrieve the latest version from the Fleet GitHub repository.
+#
+# You can, however, build the Fleet images of your own; see the repository README for
+# more information.
+echo "Retrieving image..."
+export REGISTRY="mcr.microsoft.com/aks/fleet"
+export FLEET_VERSION="${FLEET_VERSION:-$(curl "https://api.github.com/repos/Azure/fleet/tags" | jq -r '.[0].name')}"
+export MEMBER_AGENT_IMAGE="member-agent"
+export REFRESH_TOKEN_IMAGE="${REFRESH_TOKEN_NAME:-refresh-token}"
+export OUTPUT_TYPE="${OUTPUT_TYPE:-type=docker}"
+
+echo "Switching to member cluster context.."
+kubectl config use-context $MEMBER_CLUSTER_CONTEXT
+
+# Create the secret with the token extracted previously for member agent to use.
+echo "Creating secret..."
+kubectl create secret generic hub-kubeconfig-secret --from-literal=token=$TOKEN
+
+echo "Installing member-agent..."
+helm install member-agent charts/member-agent/ \
+        --set config.hubURL=$HUB_CLUSTER_ADDRESS  \
+        --set image.repository=$REGISTRY/$MEMBER_AGENT_IMAGE \
+        --set image.tag=$FLEET_VERSION \
+        --set refreshtoken.repository=$REGISTRY/$REFRESH_TOKEN_IMAGE \
+        --set refreshtoken.tag=$FLEET_VERSION \
+        --set image.pullPolicy=Always \
+        --set refreshtoken.pullPolicy=Always \
+        --set config.memberClusterName=$MEMBER_CLUSTER \
+        --set logVerbosity=5 \
+        --set namespace=fleet-system \
+        --set enableV1Alpha1APIs=false \
+        --set enableV1Beta1APIs=true
+
+kubectl get pods -A
+kubectl config use-context $HUB_CLUSTER_CONTEXT
+kubectl get membercluster $MEMBER_CLUSTER
+done
\ No newline at end of file