Improvements in Carvel and Kubernetes deployments. (#5687)

* Removed Carvel environmentVariables as default in values. * Improve probe paths to Actuator endpoints. * Improve CPU and memory requests.
spring-cloud · Feb 21, 2024 · 1ca3de8 · 1ca3de8
1 parent eb747af
commit 1ca3de8
Show file tree

Hide file tree

Showing 38 changed files with 188 additions and 142 deletions.
diff --git a/spring-cloud-dataflow-docs/src/main/asciidoc/configuration-carvel.adoc b/spring-cloud-dataflow-docs/src/main/asciidoc/configuration-carvel.adoc
@@ -51,6 +51,10 @@ These scripts assume you are connected to a Kubernetes cluster and `kubectl` is
 | [scdf-type] (oss, pro)
 | Creates `scdf-values.yml` in current directory based on `scdf-pro-values.yml` or `scdf-oss-values.yml`
 
+| carvel-import-secret.sh
+| <secret-name> <namespace> [secret-namespace] [--import|--placeholder]
+| [--plarholder] Creates place holder secret in namespace. [--import] Creates `SecretImport` for secretgen-controller.
+
 | setup-scdf-repo.sh
 | [scdf-type] (oss, pro)
 | Creates the namespace and installs the relevant Carvel package and credentials. If the optional _scdf-type_ is not provided the environmental variable `SCDF_TYPE` will be used.

diff --git a/src/carvel/config/dataflow-deployment.yml b/src/carvel/config/dataflow-deployment.yml
@@ -49,9 +49,8 @@ spec:
         ports:
         - containerPort: 9393
         startupProbe:
-          httpGet:
+          tcpSocket:
             port: 9393
-            path: #@ dataflow_liveness_path()
           failureThreshold: 4
           initialDelaySeconds: 10
           timeoutSeconds: 3

diff --git a/src/carvel/config/skipper-deployment.yml b/src/carvel/config/skipper-deployment.yml
@@ -41,24 +41,23 @@ spec:
         ports:
         - containerPort: 7577
         startupProbe:
-          httpGet:
+          tcpSocket:
             port: 7577
-            path: /actuator
           failureThreshold: 4
           initialDelaySeconds: 10
           timeoutSeconds: 3
           periodSeconds: 15
         livenessProbe:
           httpGet:
-            path: /actuator/health
+            path: /actuator/health/liveness
             port: 7577
           failureThreshold: 4
           initialDelaySeconds: 1
           timeoutSeconds: 3
           periodSeconds: 15
         readinessProbe:
           httpGet:
-            path: /actuator/info
+            path: /actuator/health/readiness
             port: 7577
           failureThreshold: 3
           initialDelaySeconds: 5

diff --git a/src/carvel/config/skipper.star b/src/carvel/config/skipper.star
@@ -7,13 +7,16 @@ load("monitoring/monitoring.star", "grafana_enabled")
 load("monitoring/monitoring.star", "prometheus_rsocket_proxy_enabled")
 load("common/common.star", "non_empty_string")
 def env_config():
-  env = ""
+  env = [] 
+  env.append("LANG=en_US.utf8")
+  env.append("LC_ALL=en_US.utf8")
+  env.append("JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8")
   if external_rabbitmq_enabled():
-    env = external_rabbitmq_env_str()
+    env.append(external_rabbitmq_env_str())
   elif external_kafka_enabled():
-    env = external_kafka_env_str()
+    env.append(external_kafka_env_str())
   end
-  return env
+  return ",".join(env)
 end
 
 def skipper_image():
@@ -32,6 +35,7 @@ def skipper_container_env():
   envs = []
   envs.extend([{"name": "LANG", "value": "en_US.utf8"}])
   envs.extend([{"name": "LC_ALL", "value": "en_US.utf8"}])
+  envs.extend([{"name": "JDK_JAVA_OPTIONS", "value": "-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8"}])
   envs.extend([{"name": "SPRING_CLOUD_CONFIG_ENABLED", "value": "false"}])
   envs.extend([{"name": "SPRING_CLOUD_KUBERNETES_CONFIG_ENABLE_API", "value": "false"}])
   envs.extend([{"name": "SPRING_CLOUD_KUBERNETES_SECRETS_ENABLE_API", "value": "false"}])

diff --git a/src/carvel/config/values/values.yml b/src/carvel/config/values/values.yml
@@ -20,8 +20,8 @@ scdf:
         cpu: ""
         memory: ""
       requests:
-        cpu: 500m
-        memory: 1024Mi
+        cpu: 1500m
+        memory: 1536Mi
     metrics:
       dashboard:
         url: ""
@@ -64,8 +64,8 @@ scdf:
         cpu: ""
         memory: ""
       requests:
-        cpu: 500m
-        memory: 1024Mi
+        cpu: 1500m
+        memory: 1536Mi
     database:
       url: ""
       username: ""

diff --git a/src/carvel/test/servers.test.ts b/src/carvel/test/servers.test.ts
@@ -252,7 +252,7 @@ describe('servers', () => {
     const container = deploymentContainer(deployment, SKIPPER_NAME);
     const envs = containerEnvValues(container);
     expect(envs).toBeTruthy();
-    expect(envs).toHaveLength(6);
+    expect(envs).toHaveLength(7);
     expect(envs).toEqual(
       expect.arrayContaining([
         expect.objectContaining({
@@ -573,11 +573,11 @@ describe('servers', () => {
     const dataflowContainer = deploymentContainer(dataflowDeployment, SCDF_SERVER_NAME);
     const skipperContainer = deploymentContainer(skipperDeployment, SKIPPER_NAME);
 
-    expect(dataflowContainer?.resources?.requests?.cpu).toBe('500m');
-    expect(dataflowContainer?.resources?.requests?.memory).toBe('1024Mi');
+    expect(dataflowContainer?.resources?.requests?.cpu).toBe('1500m');
+    expect(dataflowContainer?.resources?.requests?.memory).toBe('1536Mi');
 
-    expect(skipperContainer?.resources?.requests?.cpu).toBe('500m');
-    expect(skipperContainer?.resources?.requests?.memory).toBe('1024Mi');
+    expect(skipperContainer?.resources?.requests?.cpu).toBe('1500m');
+    expect(skipperContainer?.resources?.requests?.memory).toBe('1536Mi');
   });
 
   it('should change resources', async () => {

diff --git a/src/deploy/carvel/add-local-registry-secret.sh b/src/deploy/carvel/add-local-registry-secret.sh
@@ -21,14 +21,17 @@ SECRET_NAME=$1
 REGISTRY_NAME=$2
 REGISTRY_USER=$3
 REGISTRY_PWD=$4
+SECRET_NS=$NS
 if [ "$5" != "" ]; then
-    NS=$5
+    SECRET_NS=$5
 fi
-check_env NS
-kubectl create secret docker-registry "$SECRET_NAME" \
-    --docker-server="$REGISTRY_NAME" \
-    --docker-username="$REGISTRY_USER" \
-    --docker-password="$REGISTRY_PWD" \
-    --namespace "$NS"
+check_env SECRET_NAME
+check_env SECRET_NS
 
+#kubectl create secret docker-registry "$SECRET_NAME" \
+#    --docker-server="$REGISTRY_NAME" \
+#    --docker-username="$REGISTRY_USER" \
+#    --docker-password="$REGISTRY_PWD" \
+#    --namespace "$NS"
 
+"$SCDIR/carvel-import-secret.sh" "$SECRET_NAME" "$SECRET_NS"
diff --git a/src/deploy/carvel/carvel-add-registry-secret.sh b/src/deploy/carvel/carvel-add-registry-secret.sh
@@ -13,21 +13,35 @@ function check_env() {
 }
 
 function create_secret() {
-    echo "Create docker-registry secret $1 for $2 username=$3"
-    kubectl create secret docker-registry "$1" \
-        --docker-server="$2" \
-        --docker-username="$3" \
-        --docker-password="$4" \
-        --namespace "$5"
-    kubectl create secret docker-registry "$1" \
-            --docker-server="$2" \
-            --docker-username="$3" \
-            --docker-password="$4" \
-            --namespace "$NS"
-#    "$SCDIR/carvel-import-secret.sh" "$1" "$NS" "$5"
-    echo "Annotating $1 for image-pull-secret"
-    kubectl annotate secret "$1" --namespace "$5"  secretgen.carvel.dev/image-pull-secret=""
-    kubectl annotate secret "$1" --namespace "$NS"  secretgen.carvel.dev/image-pull-secret=""
+    SCRT_NAME=$1
+    REG_NAME=$2
+    REG_USER=$3
+    REG_PWD=$4
+    SCRT_NS=$5
+    echo "Create docker-registry secret $SCRT_NAME for $REG_NAME username=$REG_USER"
+    kubectl create secret docker-registry "$SCRT_NAME" \
+        --docker-server="$REG_NAME" \
+        --docker-username="$REG_USER" \
+        --docker-password="$REG_PWD" \
+        --namespace "$SCRT_NS"
+#    kubectl create secret docker-registry "$SCRT_NAME" \
+#            --docker-server="$REG_NAME" \
+#            --docker-username="$REG_USER" \
+#            --docker-password="$4" \
+#            --namespace "$NS"
+    echo "Annotating $SCRT_NAME for image-pull-secret"
+    # kubectl annotate secret "$SCRT_NAME" --namespace "$SCRT_NS"  secretgen.carvel.dev/image-pull-secret=""
+#    kubectl annotate secret "$1" --namespace "$NS"  secretgen.carvel.dev/image-pull-secret=""
+    echo "Exporting $SCRT_NAME from $SCRT_NS"
+    kubectl apply -f - <<EOF
+apiVersion: secretgen.carvel.dev/v1alpha1
+kind: SecretExport
+metadata:
+  name: ${SCRT_NAME}
+  namespace: ${SCRT_NS}
+spec:
+  toNamespace: '*'
+EOF
 
 }
 if [ "$4" = "" ]; then
@@ -41,6 +55,8 @@ REGISTRY_PWD=$4
 if [ "$5" != "" ]; then
     NS=$5
 fi
+check_env SECRET_NAME
+check_env REGISTRY_NAME
+check_env REGISTRY_USER
 check_env NS
 create_secret "$SECRET_NAME" "$REGISTRY_NAME" "$REGISTRY_USER" "$REGISTRY_PWD" "secrets-ns"
-
diff --git a/src/deploy/carvel/carvel-import-secret.sh b/src/deploy/carvel/carvel-import-secret.sh
@@ -3,13 +3,20 @@ if [ "$2" = "" ]; then
     echo "Argument required: <secret-name> <target-namespace>"
     exit 1
 fi
+IMPORT_TYPE=placeholder
 SECRET_NAME=$1
 NAMESPACE=$2
-if [ "$3" != "" ]; then
+if [ "$3" != "" ] && [ "$3" != "--import" ] && [ "$3" != "--placeholder" ]; then
     FROM_NAMESPACE=$3
+    shift
 else
     FROM_NAMESPACE=secret-ns
 fi
+if [ "$3" == "--import" ]; then
+    IMPORT_TYPE=import
+elif [ "$3" == "--placeholder" ]; then
+    IMPORT_TYPE=placeholder
+fi
 if [ "$SECRET_NAME" = "" ]; then
     echo "SECRET_NAME required"
     exit 2
@@ -18,9 +25,8 @@ if [ "$NAMESPACE" = "" ]; then
     echo "NAMESPACE required"
     exit 2
 fi
-
-FILE="$(mktemp).yml"
-cat >$FILE <<EOF
+if [ "$IMPORT_TYPE" == "import" ]; then
+kubectl apply -f -  <<EOF
 apiVersion: secretgen.carvel.dev/v1alpha1
 kind: SecretImport
 metadata:
@@ -29,12 +35,23 @@ metadata:
 spec:
   fromNamespace: $FROM_NAMESPACE
 EOF
-echo "Create SecretImport $SECRET_NAME from $FROM_NAMESPACE to $NAMESPACE"
-if [ "$DEBUG" = "true" ]; then
-    cat $FILE
+echo "Created SecretImport $SECRET_NAME from $FROM_NAMESPACE to $NAMESPACE"
+else
+kubectl apply -f -  <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: $SECRET_NAME
+  namespace: $NAMESPACE
+  annotations:
+    secretgen.carvel.dev/image-pull-secret: ""
+type: kubernetes.io/dockerconfigjson
+data:
+  .dockerconfigjson: e30K
+EOF
+echo "Created Placeholder Secret $SECRET_NAME in $NAMESPACE"
 fi
-kubectl apply -f $FILE
-rm -f $FILE
+
 if [ "$DEBUG" = "true" ]; then
     kubectl describe secret $SECRET_NAME --namespace $NAMESPACE
 fi
diff --git a/src/deploy/carvel/carvel-prepare-namespaces.sh b/src/deploy/carvel/carvel-prepare-namespaces.sh
@@ -39,7 +39,7 @@ if ((PRESENT > 0)); then
 fi
 kubectl create serviceaccount "$SA" --namespace $NS
 
-$SCDIR/carvel-add-registry-secret.sh scdf-metadata-default docker.io "$DOCKER_HUB_USERNAME" "$DOCKER_HUB_PASSWORD"
+$SCDIR/carvel-add-registry-secret.sh scdfmetadata docker.io "$DOCKER_HUB_USERNAME" "$DOCKER_HUB_PASSWORD"
 $SCDIR/carvel-add-registry-secret.sh reg-creds-dockerhub docker.io "$DOCKER_HUB_USERNAME" "$DOCKER_HUB_PASSWORD"
 
 

diff --git a/src/deploy/carvel/deploy-local-broker.sh b/src/deploy/carvel/deploy-local-broker.sh
@@ -47,11 +47,15 @@ export BROKER
 yq ".scdf.binder.type=\"$BINDER_NAME\"" -i ./scdf-values.yml
 
 if [ "$BROKER" = "rabbitmq" ]; then
-    yq ".scdf.binder.rabbit.host=\"rabbitmq.rabbitmq\"" -i ./scdf-values.yml
+    # RABBITMQ_HOST=$(kubectl get --namespace rabbitmq services rabbitmq | grep -F rabbitmq | awk '{ print $3 }')
+    RABBITMQ_HOST="rabbitmq.rabbitmq"
+    yq ".scdf.binder.rabbit.host=\"$RABBITMQ_HOST\"" -i ./scdf-values.yml
     yq ".scdf.binder.rabbit.port=5672" -i ./scdf-values.yml
 else
+    # KAFKA_HOST=$(kubectl get --namespace kafka services kafka | grep -F kafka | awk '{ print $3 }')
+    KAFKA_HOST="kafka.kafka"
     yq ".scdf.binder.type=\"kafka\"" -i ./scdf-values.yml
-    yq ".scdf.binder.kafka.broker.host=\"kafka-broker.kafka\"" -i ./scdf-values.yml
+    yq ".scdf.binder.kafka.broker.host=\"$KAFKA_HOST\"" -i ./scdf-values.yml
     yq ".scdf.binder.kafka.zk.host=\"kafka-zk.kafka\"" -i ./scdf-values.yml
 fi
 end_time=$(date +%s)

diff --git a/src/deploy/carvel/deploy-local-database.sh b/src/deploy/carvel/deploy-local-database.sh
@@ -37,6 +37,11 @@ set -e
 kubectl rollout status deployment --namespace "$DATABASE" $DATABASE
 set +e
 
+JDBC_URL="jdbc:$DATABASE://$DATABASE.$DATABASE/dataflow"
+"$SCDIR/configure-database.sh" dataflow $DATABASE "$JDBC_URL" $DATABASE database-username database-password
+"$SCDIR/configure-database.sh" skipper $DATABASE "$JDBC_URL" $DATABASE database-username database-password
+export DATABASE
+echo "Deployed $DATABASE. Host:$DATABASE.$DATABASE"
 FILE="$(mktemp).yml"
 cat >$FILE <<EOF
 apiVersion: secretgen.carvel.dev/v1alpha1
@@ -52,13 +57,7 @@ if [ "$DEBUG" = "true" ]; then
     cat $FILE
 fi
 kubectl apply -f $FILE
-
-JDBC_URL="jdbc:$DATABASE://$DATABASE.$DATABASE/dataflow"
-$SCDIR/configure-database.sh dataflow $DATABASE "$JDBC_URL" $DATABASE database-username database-password
-$SCDIR/configure-database.sh skipper $DATABASE "$JDBC_URL" $DATABASE database-username database-password
-"$SCDIR/carvel-import-secret.sh" "$DATABASE" "$NS" "$DATABASE"
-export DATABASE
-echo "Deployed $DATABASE. Host:$DATABASE.$DATABASE"
+"$SCDIR/carvel-import-secret.sh" "$DATABASE" "$NS" "$DATABASE" --import
 end_time=$(date +%s)
 elapsed=$((end_time - start_time))
 echo -e "Deployed $DATABASE in ${bold}$elapsed${end} seconds"
diff --git a/src/deploy/carvel/deploy-scdf.sh b/src/deploy/carvel/deploy-scdf.sh
@@ -61,6 +61,13 @@ if [ "$SKIPPER_VERSION" != "" ]; then
     yq ".scdf.skipper.image.tag=\"$SKIPPER_VERSION\"" -i ./scdf-values.yml
     echo "Overriding Skipper version=$SKIPPER_VERSION"
 fi
+
+"$SCDIR/carvel-import-secret.sh" "scdfmetadata" "$NS"
+"$SCDIR/carvel-import-secret.sh" "reg-creds-dockerhub" "$NS"
+
+if [ "$SCDF_TYPE" = "pro" ]; then
+    "$SCDIR/carvel-import-secret.sh" "reg-creds-dev-registry" "$NS"
+fi
 set +e
 $SCDIR/carvel-deploy-package.sh $APP_NAME $PACKAGE_NAME $PACKAGE_VERSION "./scdf-values.yml" "$NS"
 end_time=$(date +%s)

diff --git a/src/deploy/carvel/prepare-cluster.sh b/src/deploy/carvel/prepare-cluster.sh
@@ -2,9 +2,9 @@
 bold="\033[1m"
 dim="\033[2m"
 end="\033[0m"
-CERT_MANAGER_VERSION=v1.11.2
-SECRETGEN_CONTROLLER_VERSION=v0.14.3
-KAPP_CONTROLLER_VERSION=v0.45.1
+CERT_MANAGER_VERSION=v1.14.2
+SECRETGEN_CONTROLLER_VERSION=v0.17.0
+KAPP_CONTROLLER_VERSION=v0.50.0
 start_time=$(date +%s)
 echo "Deploying cert-manager $CERT_MANAGER_VERSION"
 kapp deploy --yes --wait --wait-check-interval 10s --app cert-manager \

diff --git a/src/deploy/carvel/prepare-local-namespace.sh b/src/deploy/carvel/prepare-local-namespace.sh
@@ -36,7 +36,7 @@ $SCDIR/add-roles.sh "system:aggregate-to-edit" "system:aggregate-to-admin" "syst
 
 kubectl create serviceaccount "$SA" --namespace $NS
 
-$SCDIR/add-local-registry-secret.sh scdf-metadata-default docker.io "$DOCKER_HUB_USERNAME" "$DOCKER_HUB_PASSWORD"
+$SCDIR/add-local-registry-secret.sh scdfmetadata docker.io "$DOCKER_HUB_USERNAME" "$DOCKER_HUB_PASSWORD"
 $SCDIR/add-local-registry-secret.sh reg-creds-dockerhub docker.io "$DOCKER_HUB_USERNAME" "$DOCKER_HUB_PASSWORD"
 
 

diff --git a/src/deploy/carvel/scdf-oss-values.yml b/src/deploy/carvel/scdf-oss-values.yml
@@ -11,17 +11,6 @@ scdf:
         value: "true"
       - name: SPRING_CLOUD_DATAFLOW_FEATURES_SCHEDULES_ENABLED
         value: "true"
-  skipper:
-    config:
-      spring:
-        cloud:
-          skipper:
-            server:
-              platform:
-                kubernetes:
-                  accounts:
-                    default:
-                      environmentVariables: 'LANG=en_US.utf8,LC_ALL=en_US.utf8,JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8'
   registry:
     secret:
       ref: "reg-creds-dockerhub"