Valk Alpine Update

apps/andi/README.md

@@ -29,7 +29,7 @@ To run ANDI locally, take the following steps:
 3.  Run the command `npm install` in the assets folder.
 4.  Move to the root ANDI folder by running the command `cd ..`. (You should now be in the same directory as this README.)
 5.  Run `MIX_ENV=integration mix docker.start` to start docker for ANDI local development. If this command times out and you see an error message like 'patiently gave up waiting for...', re-run the command. 
-6.  Start Phoenix endpoint locally with `AUTH0_CLIENT_SECRET="<auth_client_secret>" MIX_ENV=integration iex -S mix start`. Use the auth_client_secret that you obtained in the Auth0 requirements steps above.
+6.  Start Phoenix endpoint locally with `AUTH0_CLIENT_SECRET="8z_q5nB-8s44duzGc1NzjasvAjGbASk9X8hzRuIq0YbntNb1mbzE2Q3PXEpWnkxo" MIX_ENV=integration iex -S mix start`. Use the auth_client_secret that you obtained in the Auth0 requirements steps above.
 7.  Congratulations! ANDI is now running locally. Because Auth0 requires `https`, you can visit paths like `localhost` by using `https://127.0.0.1.nip.io:4443/datasets`
 
 NOTES:

apps/valkyrie/Dockerfile

@@ -1,7 +1,7 @@
 FROM smartcitiesdata:build as builder
 RUN MIX_ENV=prod mix distillery.release --name valkyrie
 
-FROM hexpm/elixir:1.10.4-erlang-23.2.7.5-alpine-3.16.0
+FROM hexpm/elixir:1.10.4-erlang-23.2.7.5-alpine-3.16.9
 ENV HOME /opt/app
 ENV REPLACE_OS_VARS=true
 WORKDIR ${HOME}

apps/valkyrie/mix.exs

@@ -4,7 +4,7 @@ defmodule Valkyrie.MixProject do
   def project do
     [
       app: :valkyrie,
-      version: "1.7.38",
+      version: "1.7.39",
       elixir: "~> 1.10",
       build_path: "../../_build",
       config_path: "../../config/config.exs",

charts/alchemist/.helmignore

@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj

charts/alchemist/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+appVersion: "1.0"
+description: Transforms data from the raw topic and writes it to transformed topic
+name: alchemist
+version: 1.0.6
+sources:
+  - https://github.com/UrbanOS-Public/smartcitiesdata/tree/master/apps/alchemist

charts/alchemist/README.md

@@ -0,0 +1,48 @@
+# alchemist
+
+![Version: 1.0.6](https://img.shields.io/badge/Version-1.0.6-informational?style=flat-square) ![AppVersion: 1.0](https://img.shields.io/badge/AppVersion-1.0-informational?style=flat-square)
+
+Transforms data from the raw topic and writes it to transformed topic
+
+## Source Code
+
+* <https://github.com/UrbanOS-Public/smartcitiesdata/tree/master/apps/alchemist>
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` |  |
+| enabled | bool | `false` |  |
+| fullnameOverride | string | `""` |  |
+| global.kafka.brokers | string | `"streaming-service-kafka-bootstrap:9092"` |  |
+| global.redis.host | string | `"redis.external-services"` |  |
+| global.redis.password | string | `""` |  |
+| global.redis.passwordSecret | string | `""` |  |
+| global.redis.port | int | `6379` |  |
+| global.redis.sslEnabled | bool | `false` |  |
+| image.majorPin | string | `""` |  |
+| image.minorPin | string | `""` |  |
+| image.pullPolicy | string | `"Always"` |  |
+| image.repository | string | `"smartcitiesdata/alchemist"` |  |
+| image.tag | string | `"development"` |  |
+| kafka.dlqTopic | string | `"streaming-dead-letters"` |  |
+| kafka.inputTopicPrefix | string | `"raw"` |  |
+| kafka.outputTopicPrefix | string | `"transformed"` |  |
+| localstack.enabled | bool | `false` |  |
+| monitoring.targetPort | int | `9002` |  |
+| nameOverride | string | `""` |  |
+| nodeSelector | object | `{}` |  |
+| processor_stages | int | `1` |  |
+| profiling_enabled | bool | `false` |  |
+| rbac.enabled | bool | `true` |  |
+| replicaCount | int | `1` |  |
+| resources.limits.cpu | string | `"300m"` |  |
+| resources.limits.memory | string | `"512Mi"` |  |
+| resources.requests.cpu | string | `"300m"` |  |
+| resources.requests.memory | string | `"512Mi"` |  |
+| service.type | string | `"ClusterIP"` |  |
+| tolerations | list | `[]` |  |
+
+----------------------------------------------
+Autogenerated from chart metadata using [helm-docs v1.12.0](https://github.com/norwoodj/helm-docs/releases/v1.12.0)

charts/alchemist/templates/_helpers.tpl

@@ -0,0 +1,32 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "helm.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "helm.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "helm.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}

charts/alchemist/templates/cluster-rbac.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.rbac.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Chart.Name }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Chart.Name }}
+rules:
+  - apiGroups: [""]
+    resources:
+      - endpoints
+    verbs:
+      - get
+      - watch
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Chart.Name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Chart.Name }}
+subjects:
+- kind: ServiceAccount
+  name: {{ .Chart.Name }}
+  namespace: {{ .Release.Namespace }}
+{{- end }}

charts/alchemist/templates/deployment.yaml

@@ -0,0 +1,94 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "helm.fullname" . }}
+  {{- if or .Values.image.majorPin .Values.image.minorPin }}
+  annotations:
+    {{- if .Values.image.majorPin }}
+    docker.major: "{{ .Values.image.majorPin }}"
+    {{- end }}
+    {{- if .Values.image.minorPin }}
+    docker.minor: "{{ .Values.image.minorPin }}"
+    {{- end }}
+  {{- end }}
+  labels:
+    app.kubernetes.io/name: {{ include "helm.name" . }}
+    helm.sh/chart: {{ include "helm.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "helm.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "helm.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+    spec:
+      {{- if .Values.rbac.enabled }}
+      serviceAccountName: {{ .Chart.Name }}
+      {{- end }}
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+          - containerPort: 9001
+          - containerPort: 4369
+          env:
+          - name: KAFKA_BROKERS
+            value: {{ .Values.global.kafka.brokers }}
+          - name: RUN_IN_KUBERNETES
+            value: "true"
+          - name: INPUT_TOPIC_PREFIX
+            value: {{.Values.kafka.inputTopicPrefix}}
+          - name: OUTPUT_TOPIC_PREFIX
+            value: {{.Values.kafka.outputTopicPrefix}}
+          - name: DLQ_TOPIC
+            value: {{.Values.kafka.dlqTopic}}
+          - name: REDIS_HOST
+            value: {{ .Values.global.redis.host }}
+          - name: REDIS_PORT
+            value: {{ quote .Values.global.redis.port }}
+          - name: REDIS_PASSWORD
+{{- if .Values.global.redis.passwordSecret }}
+            valueFrom:
+              secretKeyRef:
+                name: {{ quote .Values.global.redis.passwordSecret }}
+                key: {{ .Values.global.redis.passwordSecretKey | default .Values.global.redis.passwordSecret | quote }}
+{{- else }}
+            value: {{ .Values.global.redis.password }}
+{{- end }}
+          - name: REDIS_SSL
+            value: {{ quote .Values.global.redis.sslEnabled }}
+          - name: PROCESSOR_STAGES
+            value: {{ .Values.processor_stages | quote }}
+          - name: PROFILING_ENABLED
+            value: {{ .Values.profiling_enabled | quote }}
+          - name: METRICS_PORT
+            value: {{ quote .Values.monitoring.targetPort }}
+          - name: NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: NODE_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          resources:
+{{ toYaml .Values.resources | indent 12 }}
+    {{- with .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+    {{- end }}

charts/alchemist/templates/service.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Chart.Name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "helm.name" . }}
+  annotations:
+    prometheus.io/port: "{{ .Values.monitoring.targetPort }}"
+    prometheus.io/scrape: "true"
+spec:
+  selector:
+    app.kubernetes.io/name: {{ include "helm.name" . }}
+  ports:
+  - name: erlang-vm
+    protocol: TCP
+    port: 9001
+    targetPort: 9001
+  - name: epmd
+    protocol: TCP
+    port: 4369
+    targetPort: 4369
+  type: {{ .Values.service.type }}

charts/alchemist/values.yaml

@@ -0,0 +1,66 @@
+# alchemist is disabled by default until developement goals are met and the
+#     "topicPrefix" in reaper + valkyrie are updated to accomidate alchemist
+enabled: false
+
+global:
+  kafka:
+    brokers: streaming-service-kafka-bootstrap:9092
+  redis:
+    host: redis.external-services
+    port: 6379
+    password: ""
+    passwordSecret: ""
+    sslEnabled: false
+
+replicaCount: 1
+
+image:
+  repository: smartcitiesdata/alchemist
+  tag: development
+  pullPolicy: Always
+  majorPin: ""
+  minorPin: ""
+
+service:
+  type: ClusterIP
+
+# monitoring:
+# targetPort: 4000
+# If running locally on Minikube, set to true to deploy an AWS localstack instance
+# https://github.com/localstack/localstack
+localstack:
+  enabled: false
+
+kafka:
+  #comma delimited list of kafka seed brokersß
+  inputTopicPrefix: "raw"
+  outputTopicPrefix: "transformed"
+  dlqTopic: "streaming-dead-letters"
+
+nameOverride: ""
+fullnameOverride: ""
+
+processor_stages: 1
+
+profiling_enabled: false
+
+resources:
+  limits:
+    cpu: 300m
+    memory: 512Mi
+  requests:
+    cpu: 300m
+    memory: 512Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+rbac:
+  # Create a ServiceAccount or use default ServiceAccount if set as false
+  enabled: true
+
+monitoring:
+  targetPort: 9002

charts/andi/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+description: REST API to allow for dataset definition ingestion
+name: andi
+version: 2.3.19
+sources:
+  - https://github.com/UrbanOS-Public/smartcitiesdata/tree/master/apps/andi