From f2afd66bf685eedc17c3de2c211ba48db92b9c29 Mon Sep 17 00:00:00 2001 From: Valera Bronshtein <105710429+valeraBr@users.noreply.github.com> Date: Fri, 30 Aug 2024 16:50:56 +0300 Subject: [PATCH 01/33] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9394418..3c2b8b6 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ nats: To deploy it, run the following: ```bash -helm repo add superstream https://k8s.superstream.ai/ --force-update && helm install superstream superstream/superstream -f custom_values.yaml --create-namespace --namespace superstream --wait +helm repo add superstream-onprem https://k8s-onprem.superstream.ai/ --force-update && helm install superstream superstream-onprem/superstream-onprem -f custom_values.yaml --create-namespace --namespace superstream --wait ``` ## Parameters From 81202306021329cf21d306e05ca32e955b82236b Mon Sep 17 00:00:00 2001 From: Valera Bronshtein <105710429+valeraBr@users.noreply.github.com> Date: Fri, 30 Aug 2024 16:51:40 +0300 Subject: [PATCH 02/33] Update CODEOWNERS --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 00db701..772b52d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @idanasulin2706 @valeraBr +* @bkochauri-memphis @valeraBr From 5001a33a606f94468f621c9d83695823fab4b4d0 Mon Sep 17 00:00:00 2001 From: Beka Kotchauri Date: Mon, 2 Sep 2024 11:53:16 +0400 Subject: [PATCH 03/33] add token --- charts/superstream/charts/telegraf/templates/configmap.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/superstream/charts/telegraf/templates/configmap.yaml b/charts/superstream/charts/telegraf/templates/configmap.yaml index 64efced..da77cc8 100644 --- a/charts/superstream/charts/telegraf/templates/configmap.yaml +++ b/charts/superstream/charts/telegraf/templates/configmap.yaml @@ -78,7 +78,7 @@ data: "procid" ] [outputs.loki.http_headers] - Authorization = "$TOKEN" + Authorization = "5vG5afp5UJMiYe_DlnGdowsjduRplWB_7HyrwDWUNRApAdg26a1amtM82J6cePv46K9g4gisWc0wDwqXTxvb_A==" [[outputs.http]] data_format = "prometheusremotewrite" method = "POST" @@ -89,7 +89,7 @@ data: ] url = "https://prometheus.mgmt.superstream.ai/api/v1/write" [outputs.http.headers] - Authorization = "$TOKEN" + Authorization = "5vG5afp5UJMiYe_DlnGdowsjduRplWB_7HyrwDWUNRApAdg26a1amtM82J6cePv46K9g4gisWc0wDwqXTxvb_A==" [[inputs.syslog]] server = "udp://:6514" [inputs.syslog.tags] From 3824932253ca22abbf656df19f10365bcb14f928 Mon Sep 17 00:00:00 2001 From: Beka Kotchauri Date: Mon, 2 Sep 2024 15:04:02 +0400 Subject: [PATCH 04/33] add proxy option --- .../charts/telegraf/templates/configmap.yaml | 11 ++++++++-- charts/superstream/values.yaml | 22 ++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/charts/superstream/charts/telegraf/templates/configmap.yaml b/charts/superstream/charts/telegraf/templates/configmap.yaml index da77cc8..04e7590 100644 --- a/charts/superstream/charts/telegraf/templates/configmap.yaml +++ b/charts/superstream/charts/telegraf/templates/configmap.yaml @@ -78,7 +78,7 @@ data: "procid" ] [outputs.loki.http_headers] - Authorization = "5vG5afp5UJMiYe_DlnGdowsjduRplWB_7HyrwDWUNRApAdg26a1amtM82J6cePv46K9g4gisWc0wDwqXTxvb_A==" + Authorization = "$TOKEN" [[outputs.http]] data_format = "prometheusremotewrite" method = "POST" @@ -88,8 +88,15 @@ data: "internal" ] url = "https://prometheus.mgmt.superstream.ai/api/v1/write" + {{- if .Values.env }} + {{- range .Values.env }} + {{- if and (eq .name "HTTPS_PROXY") (ne .value "") }} + http_proxy_url = "{{ .value }}" + {{- end }} + {{- end }} + {{- end }} [outputs.http.headers] - Authorization = "5vG5afp5UJMiYe_DlnGdowsjduRplWB_7HyrwDWUNRApAdg26a1amtM82J6cePv46K9g4gisWc0wDwqXTxvb_A==" + Authorization = "$TOKEN" [[inputs.syslog]] server = "udp://:6514" [inputs.syslog.tags] diff --git a/charts/superstream/values.yaml b/charts/superstream/values.yaml index edb55d7..14ec4f3 100644 --- a/charts/superstream/values.yaml +++ b/charts/superstream/values.yaml @@ -2,9 +2,9 @@ # GLOBAL configuration for Superstream Engine ############################################################ global: - engineName: "" # Define the superstream engine name within 32 characters, excluding '.', and using only lowercase letters, numbers, '-', and '_'. - superstreamAccountId: "" # Provide the account ID associated with the deployment, which could be used for identifying resources or configurations tied to a specific account. - superstreamActivationToken: "" # Enter the activation token required for services or resources that need an initial token for activation or authentication. + engineName: "airgapped" # Define the superstream engine name within 32 characters, excluding '.', and using only lowercase letters, numbers, '-', and '_'. + superstreamAccountId: "223671996" # Provide the account ID associated with the deployment, which could be used for identifying resources or configurations tied to a specific account. + superstreamActivationToken: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2NvdW50X2luZm8iOnsiYWNjb3VudF9pZCI6IjIyMzY3MTk5NiIsImVtYWlsIjoiYmVrYUBtZW1waGlzLmRldiIsImp3dCI6ImV5SjBlWEFpT2lKS1YxUWlMQ0poYkdjaU9pSmxaREkxTlRFNUxXNXJaWGtpZlEuZXlKcWRHa2lPaUpJV2tGRVNGRkpVRVpZTjA5TlFWWlNTVmxFUzFwUFRVeE1TMFpQU0ZOWE5VZENWek0zVkVSVlEwbE5SVUZDUjBaR1dFSlJJaXdpYVdGMElqb3hOekV3TkRFMU9Ua3dMQ0pwYzNNaU9pSkJRMVpSUjBwSVZGVlJSRXcwUmpaQ1VETlVWVXRHTjB3MVVrbFVORlZLV2s1R1RsZElWVUpXUVVGUFZrTTNWRUZMU1ZSWU1sQlpNeUlzSW01aGJXVWlPaUl5TWpNMk56RTVPVFlpTENKemRXSWlPaUpWUTBRMlNWUTJSREpYV1VkV1VWQkhVbEJWVlVkVFRFZzNNazQwV0ZSVFVVUkJVRFZUVWtKR1VGbEhSVmhGVmpaVU16TlFOVlJCTnlJc0ltNWhkSE1pT25zaWNIVmlJanA3ZlN3aWMzVmlJanA3ZlN3aWMzVmljeUk2TFRFc0ltUmhkR0VpT2kweExDSndZWGxzYjJGa0lqb3RNU3dpZEhsd1pTSTZJblZ6WlhJaUxDSjJaWEp6YVc5dUlqb3lmWDAua1ZmMy1vb2V5dGVKSlZXdDFGWjhTUTBTYVVrZ3o1V25wdXRTSk1BaG5LNEV2NWhEUVhvY3Etd2VEdDY3dEk3V0xWNHoxVlRuVV91dUE3bTY2dUhhREEiLCJua2V5IjoiU1VBTk5YUTRJNktJWExWQkQ0WFQzS1hPM0dCQlA0WEczS1pST09KRVNYWE9GVU00N0VDWU1SNDZRWSIsIm9yZ2FuaXphdGlvbl9uYW1lIjoiYmVrYSJ9fQ.Ij-foKNzAKSQ4eRF98gFpF0mlNanjjDmjn_n94Bvhas" # Enter the activation token required for services or resources that need an initial token for activation or authentication. skipLocalAuthentication: true onPrem: true @@ -247,3 +247,19 @@ syslog: nodeSelector: {} tolerations: [] affinity: {} + +############################################################ +# Superstream Telegraf config +############################################################ +## If your environment uses a proxy server, uncomment the lines below and replace the URL with your proxy server's address. +# telegraf: +# env: +# - name: HTTPS_PROXY +# value: "http://your-proxy-server" +# - name: HOSTNAME +# value: "telegraf-polling-service" +# - name: TOKEN +# valueFrom: +# secretKeyRef: +# name: superstream-creds +# key: ACTIVATION_TOKEN From eb487f7510e3966cb8a09de1915b1a034a95dcca Mon Sep 17 00:00:00 2001 From: Beka Kotchauri Date: Mon, 2 Sep 2024 15:30:32 +0400 Subject: [PATCH 05/33] rm non relevant values --- charts/superstream/values.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/superstream/values.yaml b/charts/superstream/values.yaml index 14ec4f3..7ad1227 100644 --- a/charts/superstream/values.yaml +++ b/charts/superstream/values.yaml @@ -2,9 +2,9 @@ # GLOBAL configuration for Superstream Engine ############################################################ global: - engineName: "airgapped" # Define the superstream engine name within 32 characters, excluding '.', and using only lowercase letters, numbers, '-', and '_'. - superstreamAccountId: "223671996" # Provide the account ID associated with the deployment, which could be used for identifying resources or configurations tied to a specific account. - superstreamActivationToken: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2NvdW50X2luZm8iOnsiYWNjb3VudF9pZCI6IjIyMzY3MTk5NiIsImVtYWlsIjoiYmVrYUBtZW1waGlzLmRldiIsImp3dCI6ImV5SjBlWEFpT2lKS1YxUWlMQ0poYkdjaU9pSmxaREkxTlRFNUxXNXJaWGtpZlEuZXlKcWRHa2lPaUpJV2tGRVNGRkpVRVpZTjA5TlFWWlNTVmxFUzFwUFRVeE1TMFpQU0ZOWE5VZENWek0zVkVSVlEwbE5SVUZDUjBaR1dFSlJJaXdpYVdGMElqb3hOekV3TkRFMU9Ua3dMQ0pwYzNNaU9pSkJRMVpSUjBwSVZGVlJSRXcwUmpaQ1VETlVWVXRHTjB3MVVrbFVORlZLV2s1R1RsZElWVUpXUVVGUFZrTTNWRUZMU1ZSWU1sQlpNeUlzSW01aGJXVWlPaUl5TWpNMk56RTVPVFlpTENKemRXSWlPaUpWUTBRMlNWUTJSREpYV1VkV1VWQkhVbEJWVlVkVFRFZzNNazQwV0ZSVFVVUkJVRFZUVWtKR1VGbEhSVmhGVmpaVU16TlFOVlJCTnlJc0ltNWhkSE1pT25zaWNIVmlJanA3ZlN3aWMzVmlJanA3ZlN3aWMzVmljeUk2TFRFc0ltUmhkR0VpT2kweExDSndZWGxzYjJGa0lqb3RNU3dpZEhsd1pTSTZJblZ6WlhJaUxDSjJaWEp6YVc5dUlqb3lmWDAua1ZmMy1vb2V5dGVKSlZXdDFGWjhTUTBTYVVrZ3o1V25wdXRTSk1BaG5LNEV2NWhEUVhvY3Etd2VEdDY3dEk3V0xWNHoxVlRuVV91dUE3bTY2dUhhREEiLCJua2V5IjoiU1VBTk5YUTRJNktJWExWQkQ0WFQzS1hPM0dCQlA0WEczS1pST09KRVNYWE9GVU00N0VDWU1SNDZRWSIsIm9yZ2FuaXphdGlvbl9uYW1lIjoiYmVrYSJ9fQ.Ij-foKNzAKSQ4eRF98gFpF0mlNanjjDmjn_n94Bvhas" # Enter the activation token required for services or resources that need an initial token for activation or authentication. + engineName: "" # Define the superstream engine name within 32 characters, excluding '.', and using only lowercase letters, numbers, '-', and '_'. + superstreamAccountId: "" # Provide the account ID associated with the deployment, which could be used for identifying resources or configurations tied to a specific account. + superstreamActivationToken: "" # Enter the activation token required for services or resources that need an initial token for activation or authentication. skipLocalAuthentication: true onPrem: true From 239cc4d9883e0d2ea784fbd66b0fe8b24c94852b Mon Sep 17 00:00:00 2001 From: valeraBr Date: Mon, 2 Sep 2024 13:33:33 +0200 Subject: [PATCH 06/33] configure DP init contaienr --- .../superstream/templates/deployment-data-plane.yaml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/charts/superstream/templates/deployment-data-plane.yaml b/charts/superstream/templates/deployment-data-plane.yaml index 1351033..e53313d 100644 --- a/charts/superstream/templates/deployment-data-plane.yaml +++ b/charts/superstream/templates/deployment-data-plane.yaml @@ -25,20 +25,18 @@ spec: {{- include "superstream.selectorLabels" . | nindent 8 }}-data-plane spec: initContainers: - - name: check-nats-readiness + - name: check-control-plane-readiness image: {{ .Values.superstreamEngine.initContainers.image }} imagePullPolicy: IfNotPresent env: - - name: NATS_HOST - value: nats.{{ include "superstream.namespace" . }} - - name: NATS_MONITORING_PORT - value: "8222" + - name: CP_HOST + value: superstream-control-plane.{{ include "superstream.namespace" . }} command: - "sh" - "-c" - | - until nc -z $NATS_HOST.svc.cluster.local 4222 ; do - echo waiting for $NATS_HOST + until nc -z $CP_HOST.svc.cluster.local 8888 ; do + echo waiting for $CP_HOST sleep 2 done From 0c8aa8400cee97d0ba976f8f20beab5e8c34ad3a Mon Sep 17 00:00:00 2001 From: Beka Kotchauri Date: Mon, 2 Sep 2024 15:34:14 +0400 Subject: [PATCH 07/33] move telegraf section up --- charts/superstream/values.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/charts/superstream/values.yaml b/charts/superstream/values.yaml index 7ad1227..414df42 100644 --- a/charts/superstream/values.yaml +++ b/charts/superstream/values.yaml @@ -23,6 +23,22 @@ global: # global labels will be applied to all resources deployed by the chart labels: {} + +############################################################ +# Superstream Telegraf config +############################################################ +## If your environment uses a proxy server, uncomment the lines below and replace the URL with your proxy server's address. +# telegraf: +# env: +# - name: HTTPS_PROXY +# value: "http://your-proxy-server" +# - name: HOSTNAME +# value: "telegraf-polling-service" +# - name: TOKEN +# valueFrom: +# secretKeyRef: +# name: superstream-creds +# key: ACTIVATION_TOKEN ############################################################ # NATS config @@ -247,19 +263,3 @@ syslog: nodeSelector: {} tolerations: [] affinity: {} - -############################################################ -# Superstream Telegraf config -############################################################ -## If your environment uses a proxy server, uncomment the lines below and replace the URL with your proxy server's address. -# telegraf: -# env: -# - name: HTTPS_PROXY -# value: "http://your-proxy-server" -# - name: HOSTNAME -# value: "telegraf-polling-service" -# - name: TOKEN -# valueFrom: -# secretKeyRef: -# name: superstream-creds -# key: ACTIVATION_TOKEN From 440d548f74d4151882cd70a647afdd67c6dab387 Mon Sep 17 00:00:00 2001 From: Beka Kotchauri Date: Mon, 2 Sep 2024 15:51:55 +0400 Subject: [PATCH 08/33] update readme --- charts/superstream/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/superstream/values.yaml b/charts/superstream/values.yaml index 414df42..3a17365 100644 --- a/charts/superstream/values.yaml +++ b/charts/superstream/values.yaml @@ -25,7 +25,7 @@ global: labels: {} ############################################################ -# Superstream Telegraf config +# Telegraf config ############################################################ ## If your environment uses a proxy server, uncomment the lines below and replace the URL with your proxy server's address. # telegraf: From 2a3f8a56f212c9d4590ab2ef1e64d91f69f37d06 Mon Sep 17 00:00:00 2001 From: Beka Kotchauri Date: Mon, 2 Sep 2024 15:53:41 +0400 Subject: [PATCH 09/33] update readme --- charts/superstream/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/charts/superstream/README.md b/charts/superstream/README.md index c2184d5..17d06f0 100644 --- a/charts/superstream/README.md +++ b/charts/superstream/README.md @@ -35,6 +35,22 @@ global: skipLocalAuthentication: true onPrem: true +############################################################ +# Superstream Telegraf config +############################################################ +## If your environment uses a proxy server, uncomment the lines below and replace the URL with your proxy server's address. +# telegraf: +# env: +# - name: HTTPS_PROXY +# value: "http://your-proxy-server" +# - name: HOSTNAME +# value: "telegraf-polling-service" +# - name: TOKEN +# valueFrom: +# secretKeyRef: +# name: superstream-creds +# key: ACTIVATION_TOKEN + ############################################################ # NATS config ############################################################ @@ -49,6 +65,15 @@ nats: pvc: storageClassName: "" ``` +## Proxy Configuration + +If your environment requires the use of a proxy server to connect to external services, you need to add the HTTPS_PROXY variable to the Telegraf configuration. This ensures that Telegraf can route its traffic through the specified proxy. +Additionally, ensure that your proxy server allows connectivity to the following endpoints: + +(*) Prometheus: https://prometheus.mgmt.superstream.ai +(*) Loki: https://loki.mgmt.superstream.ai + +## Deployment Instructions To deploy it, run the following: ```bash From 985d5cd5a3773e1987513d9203498a035ed3cfa2 Mon Sep 17 00:00:00 2001 From: Beka Kotchauri Date: Mon, 2 Sep 2024 15:56:38 +0400 Subject: [PATCH 10/33] update readme --- charts/superstream/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/superstream/README.md b/charts/superstream/README.md index 17d06f0..e32cc38 100644 --- a/charts/superstream/README.md +++ b/charts/superstream/README.md @@ -70,8 +70,8 @@ nats: If your environment requires the use of a proxy server to connect to external services, you need to add the HTTPS_PROXY variable to the Telegraf configuration. This ensures that Telegraf can route its traffic through the specified proxy. Additionally, ensure that your proxy server allows connectivity to the following endpoints: -(*) Prometheus: https://prometheus.mgmt.superstream.ai -(*) Loki: https://loki.mgmt.superstream.ai +* **Prometheus:** https://prometheus.mgmt.superstream.ai +* **Loki:** https://loki.mgmt.superstream.ai ## Deployment Instructions From b2977ef803848ccff28bfd8745790420da60f3c4 Mon Sep 17 00:00:00 2001 From: Beka Kotchauri Date: Mon, 2 Sep 2024 15:58:04 +0400 Subject: [PATCH 11/33] update general readme --- README.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9394418..72d9979 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,23 @@ global: superstreamAccountId: "" # Provide the account ID associated with the deployment, which could be used for identifying resources or configurations tied to a specific account. superstreamActivationToken: "" # Enter the activation token required for services or resources that need an initial token for activation or authentication. skipLocalAuthentication: true - onPrem: true + onPrem: true +############################################################ +# Telegraf config +############################################################ +## If your environment uses a proxy server, uncomment the lines below and replace the URL with your proxy server's address. +# telegraf: +# env: +# - name: HTTPS_PROXY +# value: "http://your-proxy-server" +# - name: HOSTNAME +# value: "telegraf-polling-service" +# - name: TOKEN +# valueFrom: +# secretKeyRef: +# name: superstream-creds +# key: ACTIVATION_TOKEN ############################################################ # NATS config @@ -50,6 +65,15 @@ nats: pvc: storageClassName: "" ``` +## Proxy Configuration + +If your environment requires the use of a proxy server to connect to external services, you need to add the HTTPS_PROXY variable to the Telegraf configuration. This ensures that Telegraf can route its traffic through the specified proxy. +Additionally, ensure that your proxy server allows connectivity to the following endpoints: + +* **Prometheus:** https://prometheus.mgmt.superstream.ai +* **Loki:** https://loki.mgmt.superstream.ai + +## Deployment Instructions To deploy it, run the following: ```bash From 3509b4ee3adac7ca90136517bb0eec209373bf8b Mon Sep 17 00:00:00 2001 From: valeraBr Date: Mon, 2 Sep 2024 14:58:19 +0200 Subject: [PATCH 12/33] fix selectors --- .../templates/service-control-plane.yaml | 15 ++++++--------- charts/superstream/values.yaml | 4 ++++ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/charts/superstream/templates/service-control-plane.yaml b/charts/superstream/templates/service-control-plane.yaml index ec36a75..827436e 100644 --- a/charts/superstream/templates/service-control-plane.yaml +++ b/charts/superstream/templates/service-control-plane.yaml @@ -1,17 +1,14 @@ apiVersion: v1 kind: Service metadata: + name: {{ include "superstream.fullname" . }}-control-plane labels: - app.kubernetes.io/instance: superstream-control-plane - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/name: superstream-control-plane - argocd.argoproj.io/instance: superstream-control-plane - name: superstream-control-plane + {{- include "superstream.labels" . | nindent 4 }} spec: + type: {{ .Values.superstreamControlPlane.service.type }} ports: - - port: 8888 + - port: {{ .Values.superstreamControlPlane.service.port }} protocol: TCP - targetPort: 8888 + targetPort: http selector: - app.kubernetes.io/instance: superstream-control-plane - app.kubernetes.io/name: superstream-control-plane \ No newline at end of file + {{- include "superstream.selectorLabels" . | nindent 4 }}-control-plane \ No newline at end of file diff --git a/charts/superstream/values.yaml b/charts/superstream/values.yaml index a890bf3..984f746 100644 --- a/charts/superstream/values.yaml +++ b/charts/superstream/values.yaml @@ -63,6 +63,10 @@ superstreamControlPlane: encryptionSecretKey: "" superstreamActivationToken: "" useExisting: true + service: + enabled: true + type: ClusterIP + port: 8888 # Superstream Control Plane UI config From 1d62dcaed1fc83676d3ee83166e97390fbd98926 Mon Sep 17 00:00:00 2001 From: valeraBr Date: Mon, 2 Sep 2024 15:13:24 +0200 Subject: [PATCH 13/33] fix CP Host var --- charts/superstream/templates/deployment-data-plane.yaml | 2 +- charts/superstream/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/superstream/templates/deployment-data-plane.yaml b/charts/superstream/templates/deployment-data-plane.yaml index e53313d..fcd844a 100644 --- a/charts/superstream/templates/deployment-data-plane.yaml +++ b/charts/superstream/templates/deployment-data-plane.yaml @@ -81,7 +81,7 @@ spec: - name: NATS_PORT value: {{ .Values.superstreamEngine.internalNatsConnection.port | quote }} - name: CONTROL_PLANE_HOST - value: {{ .Values.superstreamEngine.controlPlane.host }} + value: nats.{{ include "superstream.namespace" . }} - name: CONTROL_PLANE_PORT value: {{ .Values.superstreamEngine.controlPlane.port | quote }} - name: SYSLOG diff --git a/charts/superstream/values.yaml b/charts/superstream/values.yaml index 984f746..1424f6b 100644 --- a/charts/superstream/values.yaml +++ b/charts/superstream/values.yaml @@ -205,7 +205,7 @@ superstreamEngine: host: "" port: 4222 controlPlane: - host: "broker.superstream.ai" + host: "" port: 4222 syslog: enabled: true From 67cde1ad9b9ddb6ed0d64de79a3853f8a67ddae1 Mon Sep 17 00:00:00 2001 From: Valera Bronshtein <105710429+valeraBr@users.noreply.github.com> Date: Mon, 2 Sep 2024 16:20:05 +0300 Subject: [PATCH 14/33] Update README.md --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3c2b8b6..671a3ec 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,14 @@ Reduce Costs and Boost Performance by 75% Without Changing a Single Component or ## Create secret with randomly generated passwords for the SSM ```yaml kubectl create secret generic superstream-creds-control-plane \ - --from-literal=postgres-password=$(openssl rand -base64 16) \ - --from-literal=password=$(openssl rand -base64 16) \ - --from-literal=repmgr-password=$(openssl rand -base64 16) \ - --from-literal=admin-password=$(openssl rand -base64 16) \ - --from-literal=superstream-admin-password=$(openssl rand -base64 16) \ - --from-literal=encryption-secret-key=$(openssl rand -base64 32) \ - --from-literal=jwt-secret-key=$(openssl rand -base64 32) \ - --from-literal=jwt-api-secret-key=$(openssl rand -base64 32) \ + --from-literal=postgres-password=$(openssl rand -base64 16 | tr -dc 'a-zA-Z0-9') \ + --from-literal=password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ + --from-literal=repmgr-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ + --from-literal=admin-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ + --from-literal=superstream-admin-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ + --from-literal=encryption-secret-key=$(openssl rand -base64 32| tr -dc 'a-zA-Z0-9') \ + --from-literal=jwt-secret-key=$(openssl rand -base64 32| tr -dc 'a-zA-Z0-9') \ + --from-literal=jwt-api-secret-key=$(openssl rand -base64 32| tr -dc 'a-zA-Z0-9') \ -n superstream ``` From 3e12ebdac9265aa8324ca92c4f9cccfc4e549ed7 Mon Sep 17 00:00:00 2001 From: Beka Kotchauri Date: Mon, 2 Sep 2024 17:29:48 +0400 Subject: [PATCH 15/33] rm non relevant values --- .../charts/telegraf/templates/deployment.yaml | 9 +++++++++ charts/superstream/charts/telegraf/values.yaml | 9 --------- charts/superstream/values.yaml | 7 ------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/charts/superstream/charts/telegraf/templates/deployment.yaml b/charts/superstream/charts/telegraf/templates/deployment.yaml index b1388c4..11eaf07 100644 --- a/charts/superstream/charts/telegraf/templates/deployment.yaml +++ b/charts/superstream/charts/telegraf/templates/deployment.yaml @@ -50,7 +50,16 @@ spec: {{ toYaml .Values.args | indent 8 }} {{- end }} env: + {{- if .Values.env }} {{ toYaml .Values.env | indent 8 }} + {{- end }} + - name: HOSTNAME + value: "telegraf-polling-service" + - name: TOKEN + valueFrom: + secretKeyRef: + name: superstream-creds + key: ACTIVATION_TOKEN {{- if .Values.envFromSecret }} envFrom: - secretRef: diff --git a/charts/superstream/charts/telegraf/values.yaml b/charts/superstream/charts/telegraf/values.yaml index bf392c7..2b3bf19 100644 --- a/charts/superstream/charts/telegraf/values.yaml +++ b/charts/superstream/charts/telegraf/values.yaml @@ -33,15 +33,6 @@ args: [] # This can be useful for auth tokens, etc. fullnameOverride: telegraf -# envFromSecret: "telegraf-tokens" -env: -- name: HOSTNAME - value: "telegraf-polling-service" -- name: TOKEN - valueFrom: - secretKeyRef: - name: superstream-creds - key: ACTIVATION_TOKEN # An older "volumeMounts" key was previously added which will likely # NOT WORK as you expect. Please use this newer configuration. diff --git a/charts/superstream/values.yaml b/charts/superstream/values.yaml index 3a17365..68b4c7b 100644 --- a/charts/superstream/values.yaml +++ b/charts/superstream/values.yaml @@ -32,13 +32,6 @@ global: # env: # - name: HTTPS_PROXY # value: "http://your-proxy-server" -# - name: HOSTNAME -# value: "telegraf-polling-service" -# - name: TOKEN -# valueFrom: -# secretKeyRef: -# name: superstream-creds -# key: ACTIVATION_TOKEN ############################################################ # NATS config From 2f3745acaf59a5b1aeac7c9498ea7fb929ce7de9 Mon Sep 17 00:00:00 2001 From: Beka Kotchauri Date: Mon, 2 Sep 2024 17:34:27 +0400 Subject: [PATCH 16/33] rm non relevant lines --- README.md | 7 ------- charts/superstream/README.md | 7 ------- 2 files changed, 14 deletions(-) diff --git a/README.md b/README.md index 72d9979..41bf0bd 100644 --- a/README.md +++ b/README.md @@ -43,13 +43,6 @@ global: # env: # - name: HTTPS_PROXY # value: "http://your-proxy-server" -# - name: HOSTNAME -# value: "telegraf-polling-service" -# - name: TOKEN -# valueFrom: -# secretKeyRef: -# name: superstream-creds -# key: ACTIVATION_TOKEN ############################################################ # NATS config diff --git a/charts/superstream/README.md b/charts/superstream/README.md index e32cc38..499f9b9 100644 --- a/charts/superstream/README.md +++ b/charts/superstream/README.md @@ -43,13 +43,6 @@ global: # env: # - name: HTTPS_PROXY # value: "http://your-proxy-server" -# - name: HOSTNAME -# value: "telegraf-polling-service" -# - name: TOKEN -# valueFrom: -# secretKeyRef: -# name: superstream-creds -# key: ACTIVATION_TOKEN ############################################################ # NATS config From df6b277979ff45cbc0560a6b3fcc0cfa6d7cfec0 Mon Sep 17 00:00:00 2001 From: valeraBr Date: Mon, 2 Sep 2024 16:12:12 +0200 Subject: [PATCH 17/33] fix port ui --- charts/superstream/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/superstream/values.yaml b/charts/superstream/values.yaml index 1424f6b..63814fc 100644 --- a/charts/superstream/values.yaml +++ b/charts/superstream/values.yaml @@ -90,7 +90,7 @@ superstreamControlPlane: service: enabled: true type: ClusterIP - port: 8080 + port: 80 From 5feda508be39674c88a129490c12d9862d11d6ac Mon Sep 17 00:00:00 2001 From: valeraBr Date: Mon, 2 Sep 2024 17:30:05 +0200 Subject: [PATCH 18/33] add notes --- charts/superstream/templates/NOTES.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/charts/superstream/templates/NOTES.txt b/charts/superstream/templates/NOTES.txt index 9fbf068..e31c368 100644 --- a/charts/superstream/templates/NOTES.txt +++ b/charts/superstream/templates/NOTES.txt @@ -27,8 +27,16 @@ https://docs.superstream.ai/getting-started/step-4-connect-your-clients Deployment Information ------------------------- +## Secrets ## +UI root username - admin +UI root Password - kubectl get secret {{ .Values.superstreamControlPlane.secret.name }} -n {{ .Release.Namespace }} -o jsonpath="{.data.superstream-admin-password}" | base64 --decode + ## Pods Deployed ## 2 - Superstream Data Plane +2 - Superstream Control Plane +1 - Superstream UI +3 - PostgreSQL +1 - PGPool 1 - Telegraf 1 - Superstream Syslog 3 - NATS \ No newline at end of file From 656eedc234cb5ca53e5069ba6138019d5135b9ad Mon Sep 17 00:00:00 2001 From: Valera Bronshtein <105710429+valeraBr@users.noreply.github.com> Date: Mon, 2 Sep 2024 23:12:34 +0300 Subject: [PATCH 19/33] Update README.md --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 671a3ec..f54f428 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,15 @@ kubectl create secret generic superstream-creds-control-plane \ --from-literal=repmgr-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ --from-literal=admin-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ --from-literal=superstream-admin-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ - --from-literal=encryption-secret-key=$(openssl rand -base64 32| tr -dc 'a-zA-Z0-9') \ - --from-literal=jwt-secret-key=$(openssl rand -base64 32| tr -dc 'a-zA-Z0-9') \ - --from-literal=jwt-api-secret-key=$(openssl rand -base64 32| tr -dc 'a-zA-Z0-9') \ + --from-literal=encryption-secret-key=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ + --from-literal=jwt-secret-key=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ + --from-literal=jwt-api-secret-key=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ -n superstream ``` +### Note: The following records should be 32 characters long + - encryption-secret-key + - jwt-secret-key + - jwt-api-secret-key ## Configure Environment Tokens From 8f37413759370559fff033f08f680679882d10ab Mon Sep 17 00:00:00 2001 From: valeraBr Date: Tue, 3 Sep 2024 14:42:26 +0200 Subject: [PATCH 20/33] add proxy to cp --- charts/superstream/README.md | 21 ++++++++++++------- charts/superstream/charts/nats/values.yaml | 1 + .../charts/telegraf/templates/configmap.yaml | 8 ++----- .../charts/telegraf/templates/deployment.yaml | 4 ++++ .../templates/deployment-control-plane.yaml | 4 ++++ charts/superstream/values.yaml | 3 +++ 6 files changed, 27 insertions(+), 14 deletions(-) diff --git a/charts/superstream/README.md b/charts/superstream/README.md index 499f9b9..b8d286f 100644 --- a/charts/superstream/README.md +++ b/charts/superstream/README.md @@ -10,16 +10,21 @@ Reduce Costs and Boost Performance by 75% Without Changing a Single Component or ## Create secret with randomly generated passwords for the SSM ```yaml kubectl create secret generic superstream-creds-control-plane \ - --from-literal=postgres-password=$(openssl rand -base64 16) \ - --from-literal=password=$(openssl rand -base64 16) \ - --from-literal=repmgr-password=$(openssl rand -base64 16) \ - --from-literal=admin-password=$(openssl rand -base64 16) \ - --from-literal=superstream-admin-password=$(openssl rand -base64 16) \ - --from-literal=encryption-secret-key=$(openssl rand -base64 32) \ - --from-literal=jwt-secret-key=$(openssl rand -base64 32) \ - --from-literal=jwt-api-secret-key=$(openssl rand -base64 32) \ + --from-literal=postgres-password=$(openssl rand -base64 16 | tr -dc 'a-zA-Z0-9') \ + --from-literal=password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ + --from-literal=repmgr-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ + --from-literal=admin-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ + --from-literal=superstream-admin-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ + --from-literal=encryption-secret-key=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ + --from-literal=jwt-secret-key=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ + --from-literal=jwt-api-secret-key=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ -n superstream ``` +### Note: The following records should be 32 characters long + - encryption-secret-key + - jwt-secret-key + - jwt-api-secret-key + ## Configure Environment Tokens diff --git a/charts/superstream/charts/nats/values.yaml b/charts/superstream/charts/nats/values.yaml index ac41259..dd69208 100644 --- a/charts/superstream/charts/nats/values.yaml +++ b/charts/superstream/charts/nats/values.yaml @@ -356,6 +356,7 @@ container: resources: requests: cpu: 100m + memory: 100Mi limits: memory: 1Gi patch: [] diff --git a/charts/superstream/charts/telegraf/templates/configmap.yaml b/charts/superstream/charts/telegraf/templates/configmap.yaml index 04e7590..9f4a39c 100644 --- a/charts/superstream/charts/telegraf/templates/configmap.yaml +++ b/charts/superstream/charts/telegraf/templates/configmap.yaml @@ -88,12 +88,8 @@ data: "internal" ] url = "https://prometheus.mgmt.superstream.ai/api/v1/write" - {{- if .Values.env }} - {{- range .Values.env }} - {{- if and (eq .name "HTTPS_PROXY") (ne .value "") }} - http_proxy_url = "{{ .value }}" - {{- end }} - {{- end }} + {{- if .Values.global.proxy.enabled }} + http_proxy_url = "{{ .Values.global.proxy.proxyUrl }}" {{- end }} [outputs.http.headers] Authorization = "$TOKEN" diff --git a/charts/superstream/charts/telegraf/templates/deployment.yaml b/charts/superstream/charts/telegraf/templates/deployment.yaml index 11eaf07..658aba9 100644 --- a/charts/superstream/charts/telegraf/templates/deployment.yaml +++ b/charts/superstream/charts/telegraf/templates/deployment.yaml @@ -53,6 +53,10 @@ spec: {{- if .Values.env }} {{ toYaml .Values.env | indent 8 }} {{- end }} + {{- if .Values.global.proxy.enabled }} + - name: HTTPS_PROXY + value: {{ .Values.global.proxy.proxyUrl }} + {{- end }} - name: HOSTNAME value: "telegraf-polling-service" - name: TOKEN diff --git a/charts/superstream/templates/deployment-control-plane.yaml b/charts/superstream/templates/deployment-control-plane.yaml index 3e3b95e..b41965a 100644 --- a/charts/superstream/templates/deployment-control-plane.yaml +++ b/charts/superstream/templates/deployment-control-plane.yaml @@ -91,6 +91,10 @@ spec: value: "8888" - name: ON_PREM value: {{ .Values.global.onPrem | quote }} + {{- if .Values.global.proxy.enabled }} + - name: HTTPS_PROXY + value: {{ .Values.global.proxy.proxyUrl }} + {{- end }} - name: SSM_ADMIN_PASSWORD valueFrom: secretKeyRef: diff --git a/charts/superstream/values.yaml b/charts/superstream/values.yaml index 287993b..834c908 100644 --- a/charts/superstream/values.yaml +++ b/charts/superstream/values.yaml @@ -23,6 +23,9 @@ global: # global labels will be applied to all resources deployed by the chart labels: {} + proxy: + enabled: true + proxyUrl: "https://your-proxy-server" ############################################################ # Telegraf config From 9defcf9946e72c8e08f769ea569496932f753d8d Mon Sep 17 00:00:00 2001 From: valeraBr Date: Tue, 3 Sep 2024 14:43:39 +0200 Subject: [PATCH 21/33] add proxy to cp --- charts/superstream/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/superstream/values.yaml b/charts/superstream/values.yaml index 834c908..e9974fc 100644 --- a/charts/superstream/values.yaml +++ b/charts/superstream/values.yaml @@ -24,7 +24,7 @@ global: # global labels will be applied to all resources deployed by the chart labels: {} proxy: - enabled: true + enabled: false proxyUrl: "https://your-proxy-server" ############################################################ From dbbeb307af8a9b951a4b2b1f9e8b98eb4e5f0ec3 Mon Sep 17 00:00:00 2001 From: valeraBr Date: Tue, 3 Sep 2024 14:46:14 +0200 Subject: [PATCH 22/33] add proxy to cp --- charts/superstream/charts/nats/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/superstream/charts/nats/values.yaml b/charts/superstream/charts/nats/values.yaml index dd69208..f5a238b 100644 --- a/charts/superstream/charts/nats/values.yaml +++ b/charts/superstream/charts/nats/values.yaml @@ -358,7 +358,7 @@ container: cpu: 100m memory: 100Mi limits: - memory: 1Gi + memory: 4Gi patch: [] ############################################################ From 57fb3d5bf27cf204b6a6107e85f6b2b0a2536df1 Mon Sep 17 00:00:00 2001 From: valeraBr Date: Tue, 3 Sep 2024 16:27:38 +0200 Subject: [PATCH 23/33] add another metrics input plugin --- README.md | 2 ++ charts/superstream/README.md | 2 ++ .../charts/telegraf/templates/configmap.yaml | 12 ++++++++++++ .../charts/telegraf/templates/deployment.yaml | 7 ++++++- charts/superstream/templates/_helpers.tpl | 1 + 5 files changed, 23 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2369696..339d794 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ kubectl create secret generic superstream-creds-control-plane \ --from-literal=repmgr-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ --from-literal=admin-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ --from-literal=superstream-admin-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ + --from-literal=control-plane-token=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ --from-literal=encryption-secret-key=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ --from-literal=jwt-secret-key=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ --from-literal=jwt-api-secret-key=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ @@ -24,6 +25,7 @@ kubectl create secret generic superstream-creds-control-plane \ - encryption-secret-key - jwt-secret-key - jwt-api-secret-key + - control-plane-token ## Configure Environment Tokens diff --git a/charts/superstream/README.md b/charts/superstream/README.md index b8d286f..d279303 100644 --- a/charts/superstream/README.md +++ b/charts/superstream/README.md @@ -15,6 +15,7 @@ kubectl create secret generic superstream-creds-control-plane \ --from-literal=repmgr-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ --from-literal=admin-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ --from-literal=superstream-admin-password=$(openssl rand -base64 16| tr -dc 'a-zA-Z0-9') \ + --from-literal=control-plane-token=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ --from-literal=encryption-secret-key=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ --from-literal=jwt-secret-key=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ --from-literal=jwt-api-secret-key=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ @@ -24,6 +25,7 @@ kubectl create secret generic superstream-creds-control-plane \ - encryption-secret-key - jwt-secret-key - jwt-api-secret-key + - control-plane-token ## Configure Environment Tokens diff --git a/charts/superstream/charts/telegraf/templates/configmap.yaml b/charts/superstream/charts/telegraf/templates/configmap.yaml index 9f4a39c..79f0538 100644 --- a/charts/superstream/charts/telegraf/templates/configmap.yaml +++ b/charts/superstream/charts/telegraf/templates/configmap.yaml @@ -118,6 +118,18 @@ data: accountId = "{{ .Values.global.superstreamAccountId }}" engineName = {{ .Values.global.engineName | quote }} + [[inputs.prometheus]] + kubernetes_label_selector = "app.kubernetes.io/component in (superstream-control-plane)" + monitor_kubernetes_pods = true + monitor_kubernetes_pods_method = "settings+annotations" + monitor_kubernetes_pods_namespace = {{ .Release.Namespace | quote}} + monitor_kubernetes_pods_port = 8888 + monitor_kubernetes_pods_path = "/monitoring/metrics" + bearer_token_string = "$CP_TOKEN" + [inputs.prometheus.tags] + accountId = "{{ .Values.global.superstreamAccountId }}" + engineName = {{ .Values.global.engineName | quote }} + [[inputs.tail]] files = ["/tmp/telegraf.log"] from_beginning = false diff --git a/charts/superstream/charts/telegraf/templates/deployment.yaml b/charts/superstream/charts/telegraf/templates/deployment.yaml index 658aba9..2257958 100644 --- a/charts/superstream/charts/telegraf/templates/deployment.yaml +++ b/charts/superstream/charts/telegraf/templates/deployment.yaml @@ -63,7 +63,12 @@ spec: valueFrom: secretKeyRef: name: superstream-creds - key: ACTIVATION_TOKEN + key: ACTIVATION_TOKEN + - name: CP_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Values.superstreamControlPlane.secret.name }} + key: control-plane-token {{- if .Values.envFromSecret }} envFrom: - secretRef: diff --git a/charts/superstream/templates/_helpers.tpl b/charts/superstream/templates/_helpers.tpl index 9bc5cbc..2e1854a 100644 --- a/charts/superstream/templates/_helpers.tpl +++ b/charts/superstream/templates/_helpers.tpl @@ -81,6 +81,7 @@ password: {{ (randAlphaNum 16) | b64enc | quote }} repmgr-password: {{ (randAlphaNum 16 ) | b64enc | quote }} admin-password: {{ (randAlphaNum 16 ) | b64enc | quote }} superstream-admin-password: {{ (randAlphaNum 16 ) | b64enc | quote }} +control-plane-token: {{ (randAlphaNum 32 ) | b64enc | quote }} encryption-secret-key: {{ (randAlphaNum 32 ) | b64enc | quote }} jwt-secret-key: {{ (randAlphaNum 32 ) | b64enc | quote }} jwt-api-secret-key: {{ (randAlphaNum 32 ) | b64enc | quote }} From 438a5d1451e1cb4cfa0ce959259eef82561f963b Mon Sep 17 00:00:00 2001 From: valeraBr Date: Tue, 3 Sep 2024 16:33:04 +0200 Subject: [PATCH 24/33] add another metrics input plugin --- charts/superstream/templates/deployment-control-plane.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/charts/superstream/templates/deployment-control-plane.yaml b/charts/superstream/templates/deployment-control-plane.yaml index b41965a..353669c 100644 --- a/charts/superstream/templates/deployment-control-plane.yaml +++ b/charts/superstream/templates/deployment-control-plane.yaml @@ -105,6 +105,11 @@ spec: secretKeyRef: name: {{ .Values.superstreamControlPlane.secret.name }} key: encryption-secret-key + - name: CONTROL_PLANE_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Values.superstreamControlPlane.secret.name }} + key: control-plane-token - name: NATS_HOST value: nats.{{ include "superstream.namespace" . }}.svc - name: NATS_PORT From 66165c8e83fa29ce3266c14ee45ce36a72a75e96 Mon Sep 17 00:00:00 2001 From: valeraBr Date: Tue, 3 Sep 2024 16:59:15 +0200 Subject: [PATCH 25/33] fix --- charts/superstream/charts/telegraf/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/superstream/charts/telegraf/templates/deployment.yaml b/charts/superstream/charts/telegraf/templates/deployment.yaml index 2257958..50ec553 100644 --- a/charts/superstream/charts/telegraf/templates/deployment.yaml +++ b/charts/superstream/charts/telegraf/templates/deployment.yaml @@ -67,7 +67,7 @@ spec: - name: CP_TOKEN valueFrom: secretKeyRef: - name: {{ .Values.superstreamControlPlane.secret.name }} + name: superstream-creds-control-plane key: control-plane-token {{- if .Values.envFromSecret }} envFrom: From a09c756244dd9e01b814a83a28307fe35ed4a5b5 Mon Sep 17 00:00:00 2001 From: valeraBr Date: Wed, 4 Sep 2024 09:17:14 +0200 Subject: [PATCH 26/33] fix readme --- charts/superstream/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/superstream/README.md b/charts/superstream/README.md index d279303..9204ec3 100644 --- a/charts/superstream/README.md +++ b/charts/superstream/README.md @@ -8,6 +8,7 @@ Reduce Costs and Boost Performance by 75% Without Changing a Single Component or ## Create secret with randomly generated passwords for the SSM +## The secret name cant be changed, will be fixed in coming release. ```yaml kubectl create secret generic superstream-creds-control-plane \ --from-literal=postgres-password=$(openssl rand -base64 16 | tr -dc 'a-zA-Z0-9') \ From 6097ee6ec821fc29ea43e387245098dcac45ad90 Mon Sep 17 00:00:00 2001 From: valeraBr Date: Wed, 4 Sep 2024 09:21:59 +0200 Subject: [PATCH 27/33] fix readme --- README.md | 4 ++++ charts/superstream/README.md | 12 ++++-------- charts/superstream/custom_values.yaml | 3 +++ charts/superstream/values.yaml | 11 ++--------- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 339d794..9de3182 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,10 @@ global: superstreamActivationToken: "" # Enter the activation token required for services or resources that need an initial token for activation or authentication. skipLocalAuthentication: true onPrem: true + ## If your environment uses a proxy server, uncomment the lines below and replace the URL with your proxy server's address. + proxy: + enabled: false + proxyUrl: "https://your-proxy-server" ############################################################ # Telegraf config diff --git a/charts/superstream/README.md b/charts/superstream/README.md index 9204ec3..c7761f0 100644 --- a/charts/superstream/README.md +++ b/charts/superstream/README.md @@ -42,15 +42,11 @@ global: superstreamActivationToken: "" # Enter the activation token required for services or resources that need an initial token for activation or authentication. skipLocalAuthentication: true onPrem: true + ## If your environment uses a proxy server, uncomment the lines below and replace the URL with your proxy server's address. + proxy: + enabled: false + proxyUrl: "https://your-proxy-server" -############################################################ -# Superstream Telegraf config -############################################################ -## If your environment uses a proxy server, uncomment the lines below and replace the URL with your proxy server's address. -# telegraf: -# env: -# - name: HTTPS_PROXY -# value: "http://your-proxy-server" ############################################################ # NATS config diff --git a/charts/superstream/custom_values.yaml b/charts/superstream/custom_values.yaml index 234306e..a8f0bc2 100644 --- a/charts/superstream/custom_values.yaml +++ b/charts/superstream/custom_values.yaml @@ -6,6 +6,9 @@ global: superstreamAccountId: "" # Provide the account ID associated with the deployment, which could be used for identifying resources or configurations tied to a specific account. superstreamActivationToken: "" # Enter the activation token required for services or resources that need an initial token for activation or authentication. skipLocalAuthentication: true + onPrem: true + + ############################################################ # NATS config ############################################################ diff --git a/charts/superstream/values.yaml b/charts/superstream/values.yaml index e9974fc..aa759f6 100644 --- a/charts/superstream/values.yaml +++ b/charts/superstream/values.yaml @@ -21,20 +21,13 @@ global: # can be overridden by individual image registry registry: - # global labels will be applied to all resources deployed by the chart + # Global labels will be applied to all resources deployed by the chart labels: {} + ## If your environment uses a proxy server, uncomment the lines below and replace the URL with your proxy server's address. proxy: enabled: false proxyUrl: "https://your-proxy-server" -############################################################ -# Telegraf config -############################################################ -## If your environment uses a proxy server, uncomment the lines below and replace the URL with your proxy server's address. -# telegraf: -# env: -# - name: HTTPS_PROXY -# value: "http://your-proxy-server" ############################################################ # NATS config From 0a4780848aff043489aeede45683f24c70c641d8 Mon Sep 17 00:00:00 2001 From: valeraBr Date: Wed, 4 Sep 2024 09:26:55 +0200 Subject: [PATCH 28/33] fix readme --- README.md | 9 +-------- charts/superstream/README.md | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 9de3182..4f26642 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Reduce Costs and Boost Performance by 75% Without Changing a Single Component or ## Create secret with randomly generated passwords for the SSM +### The secret name cant be changed, will be fixed in coming release. ```yaml kubectl create secret generic superstream-creds-control-plane \ --from-literal=postgres-password=$(openssl rand -base64 16 | tr -dc 'a-zA-Z0-9') \ @@ -45,14 +46,6 @@ global: enabled: false proxyUrl: "https://your-proxy-server" -############################################################ -# Telegraf config -############################################################ -## If your environment uses a proxy server, uncomment the lines below and replace the URL with your proxy server's address. -# telegraf: -# env: -# - name: HTTPS_PROXY -# value: "http://your-proxy-server" ############################################################ # NATS config diff --git a/charts/superstream/README.md b/charts/superstream/README.md index c7761f0..53a88dd 100644 --- a/charts/superstream/README.md +++ b/charts/superstream/README.md @@ -8,7 +8,7 @@ Reduce Costs and Boost Performance by 75% Without Changing a Single Component or ## Create secret with randomly generated passwords for the SSM -## The secret name cant be changed, will be fixed in coming release. +### The secret name cant be changed, will be fixed in coming release. ```yaml kubectl create secret generic superstream-creds-control-plane \ --from-literal=postgres-password=$(openssl rand -base64 16 | tr -dc 'a-zA-Z0-9') \ From 15a48a692d3b28d44a1db5c65046ff0b6ac661f3 Mon Sep 17 00:00:00 2001 From: valeraBr Date: Wed, 4 Sep 2024 15:56:28 +0200 Subject: [PATCH 29/33] add notes --- README.md | 10 ++++++++-- charts/superstream/README.md | 10 ++++++++-- charts/superstream/templates/NOTES.txt | 6 ++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4f26642..ca184d3 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,14 @@ The following table lists the configurable parameters of the SuperStream chart a | `global.image.pullSecretNames` | Global list of secret names to use as image pull secrets for all pod specs in the chart. Secrets must exist in the same namespace. | `[]` | | `global.image.registry` | Global registry to use for all container images in the chart. Can be overridden by individual image registry. | `""` | | `global.labels` | Global labels to use for all container images in the chart. | `""` | +| `global.onPrem` | Specifies if the deployment is for an on-premises environment. | `true` | | `nats.config.cluster.enabled` | Indicates whether the NATS cluster is enabled. | `true` | | `nats.config.jetstream.fileStore.pvc.storageClassName` | Specifies the storage class name for the Jetstream file store PVC. | `""` | +| `superstreamControlPlane.image.registry` | Docker registry to use for pulling the control plane backend service images. | `""` | +| `superstreamControlPlane.secret.useExisting` | Determines whether to use an existing secret for the control plane. | `true` | +| `superstreamControlPlane.service.port` | Port for the control plane service. | `8888` | +| `superstreamControlPlane.userInterface.image.registry` | Docker registry to use for pulling the control plane UI service images. | `""` | +| `superstreamControlPlane.userInterface.service.port` | Port for the control plane UI service. | `80` | | `superstreamEngine.releaseDate` | Release date for the backend component. | `"2024-02-22-13-03"` | | `superstreamEngine.replicaCount` | Number of replicas for the backend deployment. | `2` | | `superstreamEngine.image.repository` | Docker image repository for the backend service. | `superstreamlabs/superstream-data-plane-be` | @@ -153,7 +159,7 @@ The following table lists the configurable parameters of the SuperStream chart a | `syslog.imagePullSecrets` | Image pull secrets. | `[]` | | `syslog.service.type` | Type of service for syslog. | `ClusterIP` | | `syslog.service.port` | Port for the syslog service. | `5514` | -| `syslog.service.protocol` | Protocol for the syslog service. | `UDP` | +| `syslog.service.protocol` | Protocol used by the syslog server. | `UDP` | | `syslog.resources.limits.cpu` | CPU limit for the syslog pod. | `"100m"` | | `syslog.resources.limits.memory` | Memory limit for the syslog pod. | `"256Mi"` | | `syslog.resources.requests.cpu` | CPU request for the syslog pod. | `"50m"` | @@ -164,7 +170,7 @@ The following table lists the configurable parameters of the SuperStream chart a | `syslog.remoteSyslog.protocol` | Protocol (e.g., UDP) for the remote syslog. | `udp` | | `syslog.configMap.enabled` | Enable ConfigMap for syslog. | `true` | | `syslog.configMap.name` | Name of the ConfigMap for syslog. | `syslog-config` | -| `syslog.configMap.mountPath` | Mount path for the syslog ConfigMap. | `/config/syslog-ng.conf` | +| `syslog.configMap.mountPath` | Mount path for the syslog ConfigMap. | `/tmp/syslog-ng.conf` | | `syslog.configMap.subPath` | Specific file to mount from the ConfigMap. | `syslog-ng.conf` | | `syslog.persistence.enabled` | Enable persistence for syslog. | `false` | | `syslog.persistence.size` | Size of the persistent volume for syslog. | `"1Gi"` | diff --git a/charts/superstream/README.md b/charts/superstream/README.md index 53a88dd..c9f98a8 100644 --- a/charts/superstream/README.md +++ b/charts/superstream/README.md @@ -90,8 +90,14 @@ The following table lists the configurable parameters of the SuperStream chart a | `global.image.pullSecretNames` | Global list of secret names to use as image pull secrets for all pod specs in the chart. Secrets must exist in the same namespace. | `[]` | | `global.image.registry` | Global registry to use for all container images in the chart. Can be overridden by individual image registry. | `""` | | `global.labels` | Global labels to use for all container images in the chart. | `""` | +| `global.onPrem` | Specifies if the deployment is for an on-premises environment. | `true` | | `nats.config.cluster.enabled` | Indicates whether the NATS cluster is enabled. | `true` | | `nats.config.jetstream.fileStore.pvc.storageClassName` | Specifies the storage class name for the Jetstream file store PVC. | `""` | +| `superstreamControlPlane.image.registry` | Docker registry to use for pulling the control plane backend service images. | `""` | +| `superstreamControlPlane.secret.useExisting` | Determines whether to use an existing secret for the control plane. | `true` | +| `superstreamControlPlane.service.port` | Port for the control plane service. | `8888` | +| `superstreamControlPlane.userInterface.image.registry` | Docker registry to use for pulling the control plane UI service images. | `""` | +| `superstreamControlPlane.userInterface.service.port` | Port for the control plane UI service. | `80` | | `superstreamEngine.releaseDate` | Release date for the backend component. | `"2024-02-22-13-03"` | | `superstreamEngine.replicaCount` | Number of replicas for the backend deployment. | `2` | | `superstreamEngine.image.repository` | Docker image repository for the backend service. | `superstreamlabs/superstream-data-plane-be` | @@ -154,7 +160,7 @@ The following table lists the configurable parameters of the SuperStream chart a | `syslog.imagePullSecrets` | Image pull secrets. | `[]` | | `syslog.service.type` | Type of service for syslog. | `ClusterIP` | | `syslog.service.port` | Port for the syslog service. | `5514` | -| `syslog.service.protocol` | Protocol for the syslog service. | `UDP` | +| `syslog.service.protocol` | Protocol used by the syslog server. | `UDP` | | `syslog.resources.limits.cpu` | CPU limit for the syslog pod. | `"100m"` | | `syslog.resources.limits.memory` | Memory limit for the syslog pod. | `"256Mi"` | | `syslog.resources.requests.cpu` | CPU request for the syslog pod. | `"50m"` | @@ -165,7 +171,7 @@ The following table lists the configurable parameters of the SuperStream chart a | `syslog.remoteSyslog.protocol` | Protocol (e.g., UDP) for the remote syslog. | `udp` | | `syslog.configMap.enabled` | Enable ConfigMap for syslog. | `true` | | `syslog.configMap.name` | Name of the ConfigMap for syslog. | `syslog-config` | -| `syslog.configMap.mountPath` | Mount path for the syslog ConfigMap. | `/config/syslog-ng.conf` | +| `syslog.configMap.mountPath` | Mount path for the syslog ConfigMap. | `/tmp/syslog-ng.conf` | | `syslog.configMap.subPath` | Specific file to mount from the ConfigMap. | `syslog-ng.conf` | | `syslog.persistence.enabled` | Enable persistence for syslog. | `false` | | `syslog.persistence.size` | Size of the persistent volume for syslog. | `"1Gi"` | diff --git a/charts/superstream/templates/NOTES.txt b/charts/superstream/templates/NOTES.txt index e31c368..e70bdd2 100644 --- a/charts/superstream/templates/NOTES.txt +++ b/charts/superstream/templates/NOTES.txt @@ -31,6 +31,12 @@ Deployment Information UI root username - admin UI root Password - kubectl get secret {{ .Values.superstreamControlPlane.secret.name }} -n {{ .Release.Namespace }} -o jsonpath="{.data.superstream-admin-password}" | base64 --decode +## Next Steps ## +1. Expose the Superstream Control Plane service. It is a hard requirement to use "superstream-api" at the beginning of the configured FQDN. + Example: "superstream-api.example.com" +2. Expose the Superstream Control Plane UI service. +3. Log in to the Superstream UI and connect your first Kafka cluster. + ## Pods Deployed ## 2 - Superstream Data Plane 2 - Superstream Control Plane From 97c32adaf9b41e6fd7398571127048c52ea4ead1 Mon Sep 17 00:00:00 2001 From: valeraBr Date: Wed, 4 Sep 2024 16:32:54 +0200 Subject: [PATCH 30/33] add notes --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ca184d3..d3037ee 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ Additionally, ensure that your proxy server allows connectivity to the following * **Prometheus:** https://prometheus.mgmt.superstream.ai * **Loki:** https://loki.mgmt.superstream.ai +* **Stigg** httpsL//api.stigg.io ## Deployment Instructions From 397e7f0e18ec0aaa07d8feb632373c130819d25a Mon Sep 17 00:00:00 2001 From: Valera Bronshtein <105710429+valeraBr@users.noreply.github.com> Date: Wed, 4 Sep 2024 18:02:12 +0300 Subject: [PATCH 31/33] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d3037ee..1a291ae 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Additionally, ensure that your proxy server allows connectivity to the following * **Prometheus:** https://prometheus.mgmt.superstream.ai * **Loki:** https://loki.mgmt.superstream.ai -* **Stigg** httpsL//api.stigg.io +* **Stigg** https://api.stigg.io ## Deployment Instructions From 8a39b4236a24fc6bd5987d0eed607b2af346d302 Mon Sep 17 00:00:00 2001 From: Valera Bronshtein <105710429+valeraBr@users.noreply.github.com> Date: Wed, 4 Sep 2024 18:15:30 +0300 Subject: [PATCH 32/33] Update README.md --- README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 1a291ae..2ccef57 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,11 @@ Reduce Costs and Boost Performance by 75% Without Changing a Single Component or Your Existing Kafka! +This guide provides instructions for deploying the Superstream All-In-One distribution. -## Create secret with randomly generated passwords for the SSM -### The secret name cant be changed, will be fixed in coming release. +## Create Secrets with Randomly Generated Passwords for SSM +### The secret name `superstream-creds-control-plane` cannot be changed in the current release. This will be fixed in an upcoming release. +To create a secret for the Superstream with randomly generated passwords, run the following command: ```yaml kubectl create secret generic superstream-creds-control-plane \ --from-literal=postgres-password=$(openssl rand -base64 16 | tr -dc 'a-zA-Z0-9') \ @@ -22,7 +24,7 @@ kubectl create secret generic superstream-creds-control-plane \ --from-literal=jwt-api-secret-key=$(openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c32) \ -n superstream ``` -### Note: The following records should be 32 characters long +### Note: The following keys should have a length of 32 characters: - encryption-secret-key - jwt-secret-key - jwt-api-secret-key @@ -30,7 +32,7 @@ kubectl create secret generic superstream-creds-control-plane \ ## Configure Environment Tokens -For easiness, create `custom_values.yaml` file and edit the following values: +For a more straightforward configuration, create a `custom_values.yaml` file and edit the following values: ```yaml ############################################################ # GLOBAL configuration for Superstream Engine @@ -62,9 +64,8 @@ nats: storageClassName: "" ``` ## Proxy Configuration - -If your environment requires the use of a proxy server to connect to external services, you need to add the HTTPS_PROXY variable to the Telegraf configuration. This ensures that Telegraf can route its traffic through the specified proxy. -Additionally, ensure that your proxy server allows connectivity to the following endpoints: +If your environment requires a proxy server to connect to external services, set the global.proxy.enabled variable to true and provide the global.proxy.proxyUrl in the custom_values.yaml file. This configuration ensures that all critical services route traffic through the specified proxy. +Additionally, make sure your proxy server permits connectivity to the following endpoints: * **Prometheus:** https://prometheus.mgmt.superstream.ai * **Loki:** https://loki.mgmt.superstream.ai @@ -72,7 +73,7 @@ Additionally, ensure that your proxy server allows connectivity to the following ## Deployment Instructions -To deploy it, run the following: +To deploy the Superstream, run the following command: ```bash helm repo add superstream-onprem https://k8s-onprem.superstream.ai/ --force-update && helm install superstream superstream-onprem/superstream-onprem -f custom_values.yaml --create-namespace --namespace superstream --wait ``` From 1be6585df6e758e0d16733b849523d604efc7844 Mon Sep 17 00:00:00 2001 From: valeraBr Date: Thu, 5 Sep 2024 10:31:42 +0200 Subject: [PATCH 33/33] release prep --- charts/superstream/Chart.yaml | 4 ++-- charts/superstream/custom_values.yaml | 18 +++++++++++------- charts/superstream/templates/NOTES.txt | 6 +++++- charts/superstream/values.yaml | 7 +++---- version.conf | 2 +- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/charts/superstream/Chart.yaml b/charts/superstream/Chart.yaml index 50a9dbd..405f085 100644 --- a/charts/superstream/Chart.yaml +++ b/charts/superstream/Chart.yaml @@ -25,12 +25,12 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.1.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.0.214" +appVersion: "1.0.402" icon: https://memphis-public-files.s3.eu-central-1.amazonaws.com/superstream_logo.png dependencies: - name: nats diff --git a/charts/superstream/custom_values.yaml b/charts/superstream/custom_values.yaml index a8f0bc2..19d53d5 100644 --- a/charts/superstream/custom_values.yaml +++ b/charts/superstream/custom_values.yaml @@ -2,13 +2,17 @@ # GLOBAL configuration for Superstream Engine ############################################################ global: - engineName: "" # Define the superstream engine name within 32 characters, excluding '.', and using only lowercase letters, numbers, '-', and '_'. - superstreamAccountId: "" # Provide the account ID associated with the deployment, which could be used for identifying resources or configurations tied to a specific account. - superstreamActivationToken: "" # Enter the activation token required for services or resources that need an initial token for activation or authentication. + engineName: "" # Define the superstream engine name within 32 characters, excluding '.', and using only lowercase letters, numbers, '-', and '_'. + superstreamAccountId: "" # Provide the account ID associated with the deployment, which could be used for identifying resources or configurations tied to a specific account. + superstreamActivationToken: "" # Enter the activation token required for services or resources that need an initial token for activation or authentication. skipLocalAuthentication: true - onPrem: true + onPrem: true + ## If your environment uses a proxy server, uncomment the lines below and replace the URL with your proxy server's address. + proxy: + enabled: false + proxyUrl: "https://your-proxy-server" + - ############################################################ # NATS config ############################################################ @@ -17,8 +21,8 @@ nats: config: cluster: enabled: true -# NATS storageClass configuration. Default is blank "". +# NATS storageClass configuration. The default is blank "". jetstream: fileStore: pvc: - storageClassName: "" + storageClassName: "" \ No newline at end of file diff --git a/charts/superstream/templates/NOTES.txt b/charts/superstream/templates/NOTES.txt index e70bdd2..fbe3c7e 100644 --- a/charts/superstream/templates/NOTES.txt +++ b/charts/superstream/templates/NOTES.txt @@ -28,13 +28,17 @@ Deployment Information ------------------------- ## Secrets ## -UI root username - admin +UI root Username - admin UI root Password - kubectl get secret {{ .Values.superstreamControlPlane.secret.name }} -n {{ .Release.Namespace }} -o jsonpath="{.data.superstream-admin-password}" | base64 --decode +## Requirements ## +To be able to use the Superstream User Interface the following two FQDN records should be exposed under the same domain. + ## Next Steps ## 1. Expose the Superstream Control Plane service. It is a hard requirement to use "superstream-api" at the beginning of the configured FQDN. Example: "superstream-api.example.com" 2. Expose the Superstream Control Plane UI service. + Example: "superstream-app.example.com" 3. Log in to the Superstream UI and connect your first Kafka cluster. ## Pods Deployed ## diff --git a/charts/superstream/values.yaml b/charts/superstream/values.yaml index aa759f6..72db392 100644 --- a/charts/superstream/values.yaml +++ b/charts/superstream/values.yaml @@ -16,7 +16,6 @@ global: # secrets must exist in the same namespace # https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ pullSecretNames: - - regcred # global registry to use for all container images in the chart # can be overridden by individual image registry registry: @@ -55,7 +54,7 @@ superstreamControlPlane: # Policy for pulling the image pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "1.0.301" + tag: "1.0.402" registry: imagePullSecrets: [] @@ -84,7 +83,7 @@ superstreamControlPlane: # Policy for pulling the image pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "1.0.402" + tag: "1.0.403" registry: imagePullSecrets: [] @@ -112,7 +111,7 @@ superstreamEngine: # Policy for pulling the image pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "latest" + tag: "1.0.401" registry: imagePullSecrets: [] # Overrides for Helm's default naming conventions diff --git a/version.conf b/version.conf index 6e8bf73..6da28dd 100644 --- a/version.conf +++ b/version.conf @@ -1 +1 @@ -0.1.0 +0.1.1 \ No newline at end of file