Merge pull request #498 from desci-labs/develop

promote to main
desci-labs · Sep 18, 2024 · 7aa2ff7 · 7aa2ff7
2 parents 3294d8d + 9165b1d
commit 7aa2ff7
Show file tree

Hide file tree

Showing 120 changed files with 39,313 additions and 1,689 deletions.
diff --git a/.env.example b/.env.example
@@ -85,6 +85,7 @@ NODES_MEDIA_SERVER_URL=http://host.docker.internal:5454
 OTEL_SERVICE_NAME=
 HONEYCOMB_API_KEY=
 DISCORD_NOTIFICATIONS_WEBHOOK_URL=
+DISCORD_NOTIFICATIONS_DOI_WEBHOOK_URL=
 
 # for sending nodes cover cid to media servers
 # must match value in nodes-media/.env
@@ -156,3 +157,7 @@ ES_DB_PORT=
 ES_DB_NAME=
 ES_DB_USER=
 ES_DB_PASSWORD=
+
+
+### open Alex Database - Postgres 
+OPEN_ALEX_DATABASE_URL=postgresql://username:password@host/database?schema=openalex
diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml
@@ -47,13 +47,6 @@ jobs:
             desci-server/yarn.lock
             desci-repo/yarn.lock
 
-      - name: Set up docker-compose
-        run: |
-          sudo curl -L "https://github.com/docker/compose/releases/download/v2.18.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-          sudo chmod +x /usr/local/bin/docker-compose
-          sudo docker-compose --version
-          docker info
-
       - name: Install dependencies
         run: cd desci-models && npm i -g yarn && yarn && yarn build && cd ../desci-server && yarn --ignore-engines && cd ../desci-repo && yarn
 

diff --git a/.github/workflows/build-repo-server.yaml b/.github/workflows/build-repo-server.yaml
@@ -42,12 +42,6 @@ jobs:
   #       with:
   #         node-version: 16
 
-  #     - name: Set up docker-compose
-  #       run: |
-  #         sudo curl -L "https://github.com/docker/compose/releases/download/v2.18.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-  #         sudo docker-compose --version
-  #         docker info
-
   #     - name: Install dependencies
   #       run: cd desci-repo && yarn
 

diff --git a/.github/workflows/build-server.yaml b/.github/workflows/build-server.yaml
@@ -49,13 +49,6 @@ jobs:
             desci-models/yarn.lock
             desci-server/yarn.lock
 
-      - name: Set up docker-compose
-        run: |
-          sudo curl -L "https://github.com/docker/compose/releases/download/v2.18.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-          sudo chmod +x /usr/local/bin/docker-compose
-          sudo docker-compose --version
-          docker info
-
       - name: Install dependencies
         run: cd desci-models && npm i -g yarn && yarn && yarn build && cd ../desci-server && yarn --ignore-engines
 

diff --git a/.github/workflows/deploy-ceramic.yaml b/.github/workflows/deploy-ceramic.yaml
@@ -0,0 +1,61 @@
+on:
+  push:
+    paths:
+      - ceramic-k8s/**
+    branches:
+      - main
+      - develop
+env:
+  AWS_DEFAULT_REGION: us-east-2
+  AWS_DEFAULT_OUTPUT: json
+  AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - id: install-aws-cli
+        uses: unfor19/install-aws-cli-action@v1
+        with:
+          version: 1
+
+      - uses: prepor/action-aws-iam-authenticator@master
+      - run: aws-iam-authenticator version
+
+      - name: Install Kubectl
+        run: |
+          #$(curl -Ls https://dl.k8s.io/release/stable.txt)
+          version=v1.23.6
+          echo "using kubectl@$version"
+          curl -sLO "https://dl.k8s.io/release/$version/bin/linux/amd64/kubectl" -o kubectl
+          chmod +x kubectl
+          mv kubectl /usr/local/bin
+          mkdir $HOME/.kube
+          sudo apt-get update
+          sudo apt-get install less
+          echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
+          aws sts get-caller-identity
+
+      - name: Deploy to EKS (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          kubectl apply -f ceramic-k8s/ceramic_deployment_dev.yaml
+
+      - name: Deploy to EKS (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          kubectl apply -f ceramic-k8s/ceramic_deployment_prod.yaml
+
+      - name: Verify EKS Deployment (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          kubectl rollout status deployment/js-ceramic-dev
+
+      - name: Verify EKS Deployment (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          kubectl rollout status deployment/js-ceramic-prod
diff --git a/.github/workflows/models-build-and-test.yaml b/.github/workflows/models-build-and-test.yaml
@@ -32,13 +32,6 @@ jobs:
           cache: "yarn"
           cache-dependency-path: "desci-models/yarn.lock"
 
-      - name: Set up docker-compose
-        run: |
-          sudo curl -L "https://github.com/docker/compose/releases/download/v2.18.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-          sudo chmod +x /usr/local/bin/docker-compose
-          sudo docker-compose --version
-          docker info
-
       - name: Install dependencies
         run: cd desci-models && npm i -g yarn && yarn && yarn build
 

diff --git a/Dockerfile b/Dockerfile
@@ -2,7 +2,7 @@ FROM node:20.8.1-bullseye-slim
 
 VOLUME /root/.yarn
 
-RUN apt-get -qy update && apt-get -qy install openssl
+RUN apt-get -qy update && apt-get -qy install openssl curl
 
 RUN npm install -g npm@9.8.1
 

diff --git a/README.md b/README.md
@@ -103,7 +103,7 @@ The following guide is meant to get you running for full-stack dev on your local
 Make sure you have these packages installed on your system:
 
 - Docker
-- docker-compose
+- docker compose (comes with Docker Desktop, otherwise install the plugin package for your OS)
 - lsof (may not be available by default depending on OS)
 
 <br>

diff --git a/bootstrapCeramic.sh b/bootstrapCeramic.sh
@@ -69,7 +69,7 @@ echo "$CTX Deployment all good, probably!"
 
 if [ "$WAS_RUNNING" -eq "0" ]; then
   echo "$CTX Shutting down ceramic service..."
-  docker compose --project-name desci down
+  docker compose --project-name desci stop
 else
   echo "$CTX Leaving compose services up as they were already running when we started."
 fi 

diff --git a/ceramic-k8s/ceramic_deployment_dev.yaml b/ceramic-k8s/ceramic_deployment_dev.yaml
@@ -23,16 +23,25 @@ spec:
         vault.hashicorp.com/role: app-vault-reader
         vault.hashicorp.com/agent-inject-secret-config: secrets/desci-server/dev/db
         vault.hashicorp.com/agent-inject-template-config: |
+          {{- with secret "secrets/desci-server/ipfs-gateway-staging" -}}
+          export AWS_REGION=us-east-2
+          export AWS_ACCESS_KEY={{ .Data.AWS_ACCESS_KEY }}
+          export AWS_SECRET_KEY={{ .Data.AWS_SECRET_KEY }}
+          export AWS_DEFAULT_REGION=us-east-2
+          echo "envset"; 
+
+          {{- end -}}
           {{- with secret "secrets/desci-server/dev/db" -}}
-          echo "{\"anchor\":{\"ethereum-rpc-url\":\"{{ .Data.ceramic_rpc }}\"},\"http-api\":{\"cors-allowed-origins\":[\".*\"],\"admin-dids\":[\"did:key:z6MktbKJrMnhVJ37QFTo12911ycm2juKDUzWHDVETu9s5a9T\"]},\"ipfs\":{\"mode\":\"remote\",\"host\":\"http://public-ceramic-ipfs-dev-service-internal.default.svc.cluster.local:5001\"},\"logger\":{\"log-level\":0},\"metrics\":{\"metrics-exporter-enabled\":false,\"metrics-port\":9090},\"network\":{\"name\":\"testnet-clay\"},\"node\":{},\"state-store\":{\"mode\":\"fs\",\"local-directory\":\"/root/.ceramic/statestore\"},\"indexing\":{\"db\":\"{{ .Data.ceramic_url }}\",\"allow-queries-before-historical-sync\":true,\"models\":[]}}" > daemon.config.json
+          echo "{\"anchor\":{\"ethereum-rpc-url\":\"{{ .Data.ceramic_rpc }}\"},\"http-api\":{\"cors-allowed-origins\":[\".*\"],\"admin-dids\":[\"did:key:z6MktbKJrMnhVJ37QFTo12911ycm2juKDUzWHDVETu9s5a9T\"]},\"ipfs\":{\"mode\":\"remote\",\"host\":\"http://public-ceramic-ipfs-dev-service-internal.default.svc.cluster.local:5001\"},\"logger\":{\"log-level\":0},\"metrics\":{\"prometheus-exporter-enabled\":true,\"prometheus-exporter-port\":9464},\"network\":{\"name\":\"testnet-clay\"},\"node\":{},\"state-store\":{\"mode\":\"s3\",\"s3-bucket\":\"ceramic-node-dev-prime2\"},\"indexing\":{\"db\":\"{{ .Data.ceramic_url }}\",\"allow-queries-before-historical-sync\":true,\"enable-historical-sync\":true}}" > daemon.config.json
           export NODE_ENV=production
+          export CERAMIC_PUBSUB_QPS_LIMIT=500
           echo "envset"; 
           {{- end -}}
       labels:
         App: JsCeramicDev
     spec:
       containers:
-        - image: ceramicnetwork/js-ceramic:5.3.0
+        - image: ceramicnetwork/js-ceramic:6.4.0
           name: js-ceramic-dev
           command: ["/bin/bash", "-c"]
           args:
@@ -42,10 +51,10 @@ spec:
               name: http-api
           resources:
             limits:
-              cpu: "0.7"
+              cpu: 2
               memory: 4Gi
             requests:
-              cpu: "0.6"
+              cpu: 1
               memory: 4Gi
           # # restart pod after failureThreshold*periodSeconds total seconds
           livenessProbe:

diff --git a/ceramic-k8s/ceramic_deployment_prod.yaml b/ceramic-k8s/ceramic_deployment_prod.yaml
@@ -1,3 +1,35 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: ceramic-prod-persistent-storage
+spec:
+  capacity:
+    storage: 100Gi
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: manual
+  awsElasticBlockStore:
+    volumeID: "vol-0d7ba145f0248e211"
+    fsType: ext4
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ceramic-prod-persistent-storage-pvc
+  labels:
+    App: JsCeramicProd
+spec:
+  storageClassName: manual
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 100Gi
+
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -11,10 +43,7 @@ spec:
     matchLabels:
       App: JsCeramicProd
   strategy:
-    rollingUpdate:
-      maxSurge: 25%
-      maxUnavailable: 25%
-    type: RollingUpdate
+    type: Recreate
   template:
     metadata:
       annotations:
@@ -24,28 +53,32 @@ spec:
         vault.hashicorp.com/agent-inject-secret-config: secrets/desci-server/production/db
         vault.hashicorp.com/agent-inject-template-config: |
           {{- with secret "secrets/desci-server/production/db" -}}
-          echo "{\"anchor\":{\"auth-method\":\"did\",\"ethereum-rpc-url\":\"{{ .Data.ceramic_rpc }}\"},\"node\":{\"privateSeedUrl\":\"inplace:ed25519#{{ .Data.ceramic_cas_pkey }}\"},\"http-api\":{\"cors-allowed-origins\":[\".*\"],\"admin-dids\":[\"did:key:z6MktbKJrMnhVJ37QFTo12911ycm2juKDUzWHDVETu9s5a9T\"]},\"ipfs\":{\"mode\":\"remote\",\"host\":\"http://public-ceramic-ipfs-prod-service-internal.default.svc.cluster.local:5001\"},\"logger\":{\"log-level\":0},\"metrics\":{\"metrics-exporter-enabled\":false,\"metrics-port\":9090},\"network\":{\"name\":\"mainnet\"},\"state-store\":{\"mode\":\"fs\",\"local-directory\":\"/root/.ceramic/statestore\"},\"indexing\":{\"db\":\"{{ .Data.ceramic_url }}\",\"allow-queries-before-historical-sync\":true,\"models\":[]}}" > daemon.config.json
+          echo "{\"anchor\":{\"auth-method\":\"did\",\"ethereum-rpc-url\":\"{{ .Data.ceramic_rpc }}\"},\"node\":{\"privateSeedUrl\":\"inplace:ed25519#{{ .Data.ceramic_cas_pkey }}\"},\"http-api\":{\"cors-allowed-origins\":[\".*\"],\"admin-dids\":[\"did:key:z6MktbKJrMnhVJ37QFTo12911ycm2juKDUzWHDVETu9s5a9T\"]},\"ipfs\":{\"mode\":\"remote\",\"host\":\"http://public-ceramic-ipfs-prod-service-internal.default.svc.cluster.local:5001\"},\"logger\":{\"log-level\":0},\"metrics\":{\"prometheus-exporter-enabled\":true,\"prometheus-exporter-port\":9464},\"network\":{\"name\":\"mainnet\"},\"state-store\":{\"mode\":\"fs\",\"local-directory\":\"/root/.ceramic/statestore\"},\"indexing\":{\"db\":\"{{ .Data.ceramic_url }}\",\"allow-queries-before-historical-sync\":true,\"enable-historical-sync\":true}}" > daemon.config.json
           export NODE_ENV=production
+          export CERAMIC_PUBSUB_QPS_LIMIT=500
           echo "envset"; 
           {{- end -}}
       labels:
         App: JsCeramicProd
     spec:
       containers:
-        - image: ceramicnetwork/js-ceramic:5.3.0
+        - image: ceramicnetwork/js-ceramic:6.4.0
           name: js-ceramic-prod
           command: ["/bin/bash", "-c"]
+          volumeMounts:
+            - name: ceramic-prod-persistent-storage
+              mountPath: /root/.ceramic
           args:
             - echo "SOURCING ENV"; source /vault/secrets/config; ./packages/cli/bin/ceramic.js daemon --config daemon.config.json;
           ports:
             - containerPort: 7007
               name: http-api
           resources:
             limits:
-              cpu: "0.7"
+              cpu: 2
               memory: 4Gi
             requests:
-              cpu: "0.6"
+              cpu: 1
               memory: 4Gi
           # # restart pod after failureThreshold*periodSeconds total seconds
           livenessProbe:
@@ -61,6 +94,19 @@ spec:
               port: http-api
             failureThreshold: 60
             periodSeconds: 1
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: topology.kubernetes.io/zone
+                    operator: In
+                    values:
+                      - us-east-2c
+      volumes:
+        - name: ceramic-prod-persistent-storage
+          persistentVolumeClaim:
+            claimName: ceramic-prod-persistent-storage-pvc
 
       serviceAccountName: "vault-auth"
 ---