From 1ca3de83954e557017ce6b229c04c023bde1e14e Mon Sep 17 00:00:00 2001 From: Corneil du Plessis Date: Wed, 21 Feb 2024 18:48:15 +0200 Subject: [PATCH] Improvements in Carvel and Kubernetes deployments. (#5687) * Removed Carvel environmentVariables as default in values. * Improve probe paths to Actuator endpoints. * Improve CPU and memory requests. --- .../main/asciidoc/configuration-carvel.adoc | 4 ++ src/carvel/config/dataflow-deployment.yml | 3 +- src/carvel/config/skipper-deployment.yml | 7 ++- src/carvel/config/skipper.star | 12 +++-- src/carvel/config/values/values.yml | 8 ++-- src/carvel/test/servers.test.ts | 10 ++-- .../carvel/add-local-registry-secret.sh | 17 ++++--- .../carvel/carvel-add-registry-secret.sh | 48 ++++++++++++------- src/deploy/carvel/carvel-import-secret.sh | 35 ++++++++++---- .../carvel/carvel-prepare-namespaces.sh | 2 +- src/deploy/carvel/deploy-local-broker.sh | 8 +++- src/deploy/carvel/deploy-local-database.sh | 13 +++-- src/deploy/carvel/deploy-scdf.sh | 7 +++ src/deploy/carvel/prepare-cluster.sh | 6 +-- src/deploy/carvel/prepare-local-namespace.sh | 2 +- src/deploy/carvel/scdf-oss-values.yml | 11 ----- src/deploy/carvel/scdf-pro-values.yml | 12 +---- src/deploy/carvel/setup-scdf-repo.sh | 3 +- src/deploy/k8s/configure-k8s.sh | 4 +- src/deploy/k8s/deploy-scdf.sh | 2 +- src/deploy/k8s/yaml/server-config.yaml | 3 +- .../k8s/yaml/server-deployment-pro.yaml | 14 ++---- src/deploy/k8s/yaml/server-deployment.yaml | 8 ++-- src/deploy/k8s/yaml/skipper-config-kafka.yaml | 2 +- .../k8s/yaml/skipper-config-rabbit.yaml | 2 +- .../k8s/yaml/skipper-deployment-pro.yaml | 8 ++-- src/deploy/k8s/yaml/skipper-deployment.yaml | 8 ++-- src/deploy/shell/deploy-httplogger.sh | 13 +++++ src/deploy/versions.yaml | 2 +- src/docker-compose/docker-compose-kafka.yml | 2 +- .../mariadb/mariadb-deployment.yaml | 4 ++ .../postgresql/postgresql-deployment.yaml | 2 +- .../prometheus-proxy-deployment.yaml | 7 +-- src/kubernetes/server/server-deployment.yaml | 12 ++--- .../skipper/skipper-deployment.yaml | 6 +-- .../grafana/grafana-deployment.yaml | 7 +-- .../kubernetes/server/server-deployment.yaml | 12 +++-- .../skipper/skipper-deployment.yaml | 4 +- 38 files changed, 188 insertions(+), 142 deletions(-) create mode 100755 src/deploy/shell/deploy-httplogger.sh diff --git a/spring-cloud-dataflow-docs/src/main/asciidoc/configuration-carvel.adoc b/spring-cloud-dataflow-docs/src/main/asciidoc/configuration-carvel.adoc index 622ffe85ec..f558fa89fb 100644 --- a/spring-cloud-dataflow-docs/src/main/asciidoc/configuration-carvel.adoc +++ b/spring-cloud-dataflow-docs/src/main/asciidoc/configuration-carvel.adoc @@ -51,6 +51,10 @@ These scripts assume you are connected to a Kubernetes cluster and `kubectl` is | [scdf-type] (oss, pro) | Creates `scdf-values.yml` in current directory based on `scdf-pro-values.yml` or `scdf-oss-values.yml` +| carvel-import-secret.sh +| [secret-namespace] [--import|--placeholder] +| [--plarholder] Creates place holder secret in namespace. [--import] Creates `SecretImport` for secretgen-controller. + | setup-scdf-repo.sh | [scdf-type] (oss, pro) | Creates the namespace and installs the relevant Carvel package and credentials. If the optional _scdf-type_ is not provided the environmental variable `SCDF_TYPE` will be used. diff --git a/src/carvel/config/dataflow-deployment.yml b/src/carvel/config/dataflow-deployment.yml index 72e51f0eaf..a51378d678 100644 --- a/src/carvel/config/dataflow-deployment.yml +++ b/src/carvel/config/dataflow-deployment.yml @@ -49,9 +49,8 @@ spec: ports: - containerPort: 9393 startupProbe: - httpGet: + tcpSocket: port: 9393 - path: #@ dataflow_liveness_path() failureThreshold: 4 initialDelaySeconds: 10 timeoutSeconds: 3 diff --git a/src/carvel/config/skipper-deployment.yml b/src/carvel/config/skipper-deployment.yml index e41e4be00f..b6e6587c20 100644 --- a/src/carvel/config/skipper-deployment.yml +++ b/src/carvel/config/skipper-deployment.yml @@ -41,16 +41,15 @@ spec: ports: - containerPort: 7577 startupProbe: - httpGet: + tcpSocket: port: 7577 - path: /actuator failureThreshold: 4 initialDelaySeconds: 10 timeoutSeconds: 3 periodSeconds: 15 livenessProbe: httpGet: - path: /actuator/health + path: /actuator/health/liveness port: 7577 failureThreshold: 4 initialDelaySeconds: 1 @@ -58,7 +57,7 @@ spec: periodSeconds: 15 readinessProbe: httpGet: - path: /actuator/info + path: /actuator/health/readiness port: 7577 failureThreshold: 3 initialDelaySeconds: 5 diff --git a/src/carvel/config/skipper.star b/src/carvel/config/skipper.star index 05ce677ec7..48e3bd03a1 100644 --- a/src/carvel/config/skipper.star +++ b/src/carvel/config/skipper.star @@ -7,13 +7,16 @@ load("monitoring/monitoring.star", "grafana_enabled") load("monitoring/monitoring.star", "prometheus_rsocket_proxy_enabled") load("common/common.star", "non_empty_string") def env_config(): - env = "" + env = [] + env.append("LANG=en_US.utf8") + env.append("LC_ALL=en_US.utf8") + env.append("JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8") if external_rabbitmq_enabled(): - env = external_rabbitmq_env_str() + env.append(external_rabbitmq_env_str()) elif external_kafka_enabled(): - env = external_kafka_env_str() + env.append(external_kafka_env_str()) end - return env + return ",".join(env) end def skipper_image(): @@ -32,6 +35,7 @@ def skipper_container_env(): envs = [] envs.extend([{"name": "LANG", "value": "en_US.utf8"}]) envs.extend([{"name": "LC_ALL", "value": "en_US.utf8"}]) + envs.extend([{"name": "JDK_JAVA_OPTIONS", "value": "-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8"}]) envs.extend([{"name": "SPRING_CLOUD_CONFIG_ENABLED", "value": "false"}]) envs.extend([{"name": "SPRING_CLOUD_KUBERNETES_CONFIG_ENABLE_API", "value": "false"}]) envs.extend([{"name": "SPRING_CLOUD_KUBERNETES_SECRETS_ENABLE_API", "value": "false"}]) diff --git a/src/carvel/config/values/values.yml b/src/carvel/config/values/values.yml index 1118534c6b..af1b65fad0 100644 --- a/src/carvel/config/values/values.yml +++ b/src/carvel/config/values/values.yml @@ -20,8 +20,8 @@ scdf: cpu: "" memory: "" requests: - cpu: 500m - memory: 1024Mi + cpu: 1500m + memory: 1536Mi metrics: dashboard: url: "" @@ -64,8 +64,8 @@ scdf: cpu: "" memory: "" requests: - cpu: 500m - memory: 1024Mi + cpu: 1500m + memory: 1536Mi database: url: "" username: "" diff --git a/src/carvel/test/servers.test.ts b/src/carvel/test/servers.test.ts index 552820a26f..85cc7aee80 100644 --- a/src/carvel/test/servers.test.ts +++ b/src/carvel/test/servers.test.ts @@ -252,7 +252,7 @@ describe('servers', () => { const container = deploymentContainer(deployment, SKIPPER_NAME); const envs = containerEnvValues(container); expect(envs).toBeTruthy(); - expect(envs).toHaveLength(6); + expect(envs).toHaveLength(7); expect(envs).toEqual( expect.arrayContaining([ expect.objectContaining({ @@ -573,11 +573,11 @@ describe('servers', () => { const dataflowContainer = deploymentContainer(dataflowDeployment, SCDF_SERVER_NAME); const skipperContainer = deploymentContainer(skipperDeployment, SKIPPER_NAME); - expect(dataflowContainer?.resources?.requests?.cpu).toBe('500m'); - expect(dataflowContainer?.resources?.requests?.memory).toBe('1024Mi'); + expect(dataflowContainer?.resources?.requests?.cpu).toBe('1500m'); + expect(dataflowContainer?.resources?.requests?.memory).toBe('1536Mi'); - expect(skipperContainer?.resources?.requests?.cpu).toBe('500m'); - expect(skipperContainer?.resources?.requests?.memory).toBe('1024Mi'); + expect(skipperContainer?.resources?.requests?.cpu).toBe('1500m'); + expect(skipperContainer?.resources?.requests?.memory).toBe('1536Mi'); }); it('should change resources', async () => { diff --git a/src/deploy/carvel/add-local-registry-secret.sh b/src/deploy/carvel/add-local-registry-secret.sh index 84909d7a53..66b766bcab 100755 --- a/src/deploy/carvel/add-local-registry-secret.sh +++ b/src/deploy/carvel/add-local-registry-secret.sh @@ -21,14 +21,17 @@ SECRET_NAME=$1 REGISTRY_NAME=$2 REGISTRY_USER=$3 REGISTRY_PWD=$4 +SECRET_NS=$NS if [ "$5" != "" ]; then - NS=$5 + SECRET_NS=$5 fi -check_env NS -kubectl create secret docker-registry "$SECRET_NAME" \ - --docker-server="$REGISTRY_NAME" \ - --docker-username="$REGISTRY_USER" \ - --docker-password="$REGISTRY_PWD" \ - --namespace "$NS" +check_env SECRET_NAME +check_env SECRET_NS +#kubectl create secret docker-registry "$SECRET_NAME" \ +# --docker-server="$REGISTRY_NAME" \ +# --docker-username="$REGISTRY_USER" \ +# --docker-password="$REGISTRY_PWD" \ +# --namespace "$NS" +"$SCDIR/carvel-import-secret.sh" "$SECRET_NAME" "$SECRET_NS" diff --git a/src/deploy/carvel/carvel-add-registry-secret.sh b/src/deploy/carvel/carvel-add-registry-secret.sh index 5a1caa948f..bc8d69676a 100755 --- a/src/deploy/carvel/carvel-add-registry-secret.sh +++ b/src/deploy/carvel/carvel-add-registry-secret.sh @@ -13,21 +13,35 @@ function check_env() { } function create_secret() { - echo "Create docker-registry secret $1 for $2 username=$3" - kubectl create secret docker-registry "$1" \ - --docker-server="$2" \ - --docker-username="$3" \ - --docker-password="$4" \ - --namespace "$5" - kubectl create secret docker-registry "$1" \ - --docker-server="$2" \ - --docker-username="$3" \ - --docker-password="$4" \ - --namespace "$NS" -# "$SCDIR/carvel-import-secret.sh" "$1" "$NS" "$5" - echo "Annotating $1 for image-pull-secret" - kubectl annotate secret "$1" --namespace "$5" secretgen.carvel.dev/image-pull-secret="" - kubectl annotate secret "$1" --namespace "$NS" secretgen.carvel.dev/image-pull-secret="" + SCRT_NAME=$1 + REG_NAME=$2 + REG_USER=$3 + REG_PWD=$4 + SCRT_NS=$5 + echo "Create docker-registry secret $SCRT_NAME for $REG_NAME username=$REG_USER" + kubectl create secret docker-registry "$SCRT_NAME" \ + --docker-server="$REG_NAME" \ + --docker-username="$REG_USER" \ + --docker-password="$REG_PWD" \ + --namespace "$SCRT_NS" +# kubectl create secret docker-registry "$SCRT_NAME" \ +# --docker-server="$REG_NAME" \ +# --docker-username="$REG_USER" \ +# --docker-password="$4" \ +# --namespace "$NS" + echo "Annotating $SCRT_NAME for image-pull-secret" + # kubectl annotate secret "$SCRT_NAME" --namespace "$SCRT_NS" secretgen.carvel.dev/image-pull-secret="" +# kubectl annotate secret "$1" --namespace "$NS" secretgen.carvel.dev/image-pull-secret="" + echo "Exporting $SCRT_NAME from $SCRT_NS" + kubectl apply -f - < " exit 1 fi +IMPORT_TYPE=placeholder SECRET_NAME=$1 NAMESPACE=$2 -if [ "$3" != "" ]; then +if [ "$3" != "" ] && [ "$3" != "--import" ] && [ "$3" != "--placeholder" ]; then FROM_NAMESPACE=$3 + shift else FROM_NAMESPACE=secret-ns fi +if [ "$3" == "--import" ]; then + IMPORT_TYPE=import +elif [ "$3" == "--placeholder" ]; then + IMPORT_TYPE=placeholder +fi if [ "$SECRET_NAME" = "" ]; then echo "SECRET_NAME required" exit 2 @@ -18,9 +25,8 @@ if [ "$NAMESPACE" = "" ]; then echo "NAMESPACE required" exit 2 fi - -FILE="$(mktemp).yml" -cat >$FILE < 0)); then fi kubectl create serviceaccount "$SA" --namespace $NS -$SCDIR/carvel-add-registry-secret.sh scdf-metadata-default docker.io "$DOCKER_HUB_USERNAME" "$DOCKER_HUB_PASSWORD" +$SCDIR/carvel-add-registry-secret.sh scdfmetadata docker.io "$DOCKER_HUB_USERNAME" "$DOCKER_HUB_PASSWORD" $SCDIR/carvel-add-registry-secret.sh reg-creds-dockerhub docker.io "$DOCKER_HUB_USERNAME" "$DOCKER_HUB_PASSWORD" diff --git a/src/deploy/carvel/deploy-local-broker.sh b/src/deploy/carvel/deploy-local-broker.sh index c75b320fa7..79b313b826 100755 --- a/src/deploy/carvel/deploy-local-broker.sh +++ b/src/deploy/carvel/deploy-local-broker.sh @@ -47,11 +47,15 @@ export BROKER yq ".scdf.binder.type=\"$BINDER_NAME\"" -i ./scdf-values.yml if [ "$BROKER" = "rabbitmq" ]; then - yq ".scdf.binder.rabbit.host=\"rabbitmq.rabbitmq\"" -i ./scdf-values.yml + # RABBITMQ_HOST=$(kubectl get --namespace rabbitmq services rabbitmq | grep -F rabbitmq | awk '{ print $3 }') + RABBITMQ_HOST="rabbitmq.rabbitmq" + yq ".scdf.binder.rabbit.host=\"$RABBITMQ_HOST\"" -i ./scdf-values.yml yq ".scdf.binder.rabbit.port=5672" -i ./scdf-values.yml else + # KAFKA_HOST=$(kubectl get --namespace kafka services kafka | grep -F kafka | awk '{ print $3 }') + KAFKA_HOST="kafka.kafka" yq ".scdf.binder.type=\"kafka\"" -i ./scdf-values.yml - yq ".scdf.binder.kafka.broker.host=\"kafka-broker.kafka\"" -i ./scdf-values.yml + yq ".scdf.binder.kafka.broker.host=\"$KAFKA_HOST\"" -i ./scdf-values.yml yq ".scdf.binder.kafka.zk.host=\"kafka-zk.kafka\"" -i ./scdf-values.yml fi end_time=$(date +%s) diff --git a/src/deploy/carvel/deploy-local-database.sh b/src/deploy/carvel/deploy-local-database.sh index 0887a65036..50599638b6 100755 --- a/src/deploy/carvel/deploy-local-database.sh +++ b/src/deploy/carvel/deploy-local-database.sh @@ -37,6 +37,11 @@ set -e kubectl rollout status deployment --namespace "$DATABASE" $DATABASE set +e +JDBC_URL="jdbc:$DATABASE://$DATABASE.$DATABASE/dataflow" +"$SCDIR/configure-database.sh" dataflow $DATABASE "$JDBC_URL" $DATABASE database-username database-password +"$SCDIR/configure-database.sh" skipper $DATABASE "$JDBC_URL" $DATABASE database-username database-password +export DATABASE +echo "Deployed $DATABASE. Host:$DATABASE.$DATABASE" FILE="$(mktemp).yml" cat >$FILE < deploy-httplogger.shell <