Merge branch 'develop' of https://github.com/desci-labs/nodes

.env.example

@@ -93,8 +93,8 @@ ORCID_CLIENT_SECRET=
 REPO_SERVER_URL=http://host.docker.internal:5484
 REPO_SERVICE_SECRET_KEY=secretrepo
 
-# To run bootstrapCeramic.sh, clone `@desci-labs/desci-codex` and put the path here
-CODEX_REPO_PATH=
+# To run bootstrapCeramic.sh, you need the admin seed for the model IDs to be the same
+CERAMIC_ADMIN_SEED=
 
 # ISOLATED MEDIA SERVER
 ISOLATED_MEDIA_SERVER_URL=http://media_isolated:7771
@@ -116,3 +116,10 @@ ETHEREUM_RPC_URL=http://host.docker.internal:8545
 
 # Use this for Sepolia testnet
 # ETHEREUM_RPC_URL=https://eth-sepolia.g.alchemy.com/v2/demo
+
+DPID_URL_OVERRIDE=https://dev-beta.dpid.org
+
+# Set to true if you want to mute the publish worker in local dev
+MUTE_PUBLISH_WORKER=false
+# SingleNodeLockServce
+MAX_LOCK_TIME=3600 # 1 hour

.github/workflows/build-reverse-proxy.yaml

@@ -0,0 +1,112 @@
+# build.yml
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - reverse-proxy/**
+    branches:
+      - develop
+      - main
+
+# https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
+env:
+  AWS_DEFAULT_REGION: us-east-2
+  AWS_DEFAULT_OUTPUT: json
+  AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  CONTAINER_IMAGE: reverse-proxy
+  DOCKER_BUILDKIT: 1
+
+jobs:
+  build-and-push:
+    name: Build and deploy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: hashicorp/setup-terraform@v1
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Add steps here like linting, testing, minification, etc.
+      - id: install-aws-cli
+        uses: unfor19/install-aws-cli-action@v1
+        with:
+          version: 1
+
+      - uses: prepor/action-aws-iam-authenticator@master
+      - run: aws-iam-authenticator version
+
+      - name: Install Kubectl
+        run: |
+          #$(curl -Ls https://dl.k8s.io/release/stable.txt)
+          version=v1.23.6
+          echo "using kubectl@$version"
+          curl -sLO "https://dl.k8s.io/release/$version/bin/linux/amd64/kubectl" -o kubectl
+          chmod +x kubectl
+          mv kubectl /usr/local/bin
+          mkdir $HOME/.kube
+          sudo apt-get update
+          sudo apt-get install less
+          echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
+          aws sts get-caller-identity
+          kubectl describe deployments
+
+      - name: Build and tag the image (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          # Build and tag the image
+          docker build \
+            -t $CONTAINER_IMAGE-dev:latest \
+            -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev \
+             ./$CONTAINER_IMAGE
+
+      - name: Build and tag the image (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          # Build and tag the image
+          docker build \
+            -t $CONTAINER_IMAGE-prod:latest \
+            -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod \
+             ./$CONTAINER_IMAGE
+
+      - name: Push (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          # Push image to AWS ECR
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker tag $CONTAINER_IMAGE-dev:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
+          docker tag $CONTAINER_IMAGE-dev:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
+
+      - name: Push (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          # Push image to AWS ECR
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker tag $CONTAINER_IMAGE-prod:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:${{ github.sha }}
+          docker tag $CONTAINER_IMAGE-prod:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:latest
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:${{ github.sha }}
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:latest
+
+      - name: Deploy to EKS (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: | # defaults to latest kubectl binary version
+          kubectl apply -f $CONTAINER_IMAGE/kubernetes/deployment_dev.yaml
+          kubectl set image deployment/reverse-proxy-dev reverse-proxy-dev=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }} --record
+
+      - name: Deploy to EKS (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: | # defaults to latest kubectl binary version
+          kubectl apply -f $CONTAINER_IMAGE/kubernetes/deployment_prod.yaml
+          kubectl set image deployment/reverse-proxy-prod reverse-proxy-prod=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:${{ github.sha }} --record
+
+      - name: Verify EKS Deployment (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          kubectl rollout status deployment/reverse-proxy-dev
+
+      - name: Verify EKS Deployment (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          kubectl rollout status deployment/reverse-proxy-prod

.github/workflows/deploy-staging-services.yaml

@@ -257,3 +257,61 @@ jobs:
       - name: Verify EKS Deployment (STAGING)'
         run: |
           kubectl rollout status deployment/desci-server-staging
+
+  build-reverse-proxy:
+    name: Build and deploy reverse-proxy STAGING
+    runs-on: ubuntu-latest
+    steps:
+      - uses: hashicorp/setup-terraform@v1
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Add steps here like linting, testing, minification, etc.
+      - id: install-aws-cli
+        uses: unfor19/install-aws-cli-action@v1
+        with:
+          version: 1
+
+      - uses: prepor/action-aws-iam-authenticator@master
+      - run: aws-iam-authenticator version
+
+      - name: Install Kubectl
+        run: |
+          #$(curl -Ls https://dl.k8s.io/release/stable.txt)
+          version=v1.23.6
+          echo "using kubectl@$version"
+          curl -sLO "https://dl.k8s.io/release/$version/bin/linux/amd64/kubectl" -o kubectl
+          chmod +x kubectl
+          mv kubectl /usr/local/bin
+          mkdir $HOME/.kube
+          sudo apt-get update
+          sudo apt-get install less
+          echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
+          aws sts get-caller-identity
+          kubectl describe deployments
+
+      - name: Build and tag the image (STAGING)
+        run: |
+          # Build and tag the image
+          docker build \
+            -t $CONTAINER_IMAGE-staging:latest \
+            -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-staging \
+             ./$CONTAINER_IMAGE
+
+      - name: Push (STAGING)
+        run: |
+          # Push image to AWS ECR
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker tag $CONTAINER_IMAGE-staging:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-staging:${{ github.sha }}
+          docker tag $CONTAINER_IMAGE-staging:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-staging:latest
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-staging:${{ github.sha }}
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-staging:latest
+      
+      - name: Deploy to EKS (STAGING)
+        run: | # defaults to latest kubectl binary version
+          kubectl apply -f $CONTAINER_IMAGE/deployment.yaml
+          kubectl set image deployment/reverse-proxy-staging reverse-proxy-staging=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-staging:${{ github.sha }} --record
+
+      - name: Verify EKS Deployment (STAGING)
+        run: |
+          kubectl rollout status deployment/reverse-proxy-staging

.gitignore

@@ -15,4 +15,5 @@ ipfs-data/
 local-data/
 node-modules/
 node_modules/*
-.idea
+.idea
+.composedbRuntimeDefinition.json

.nvmrc

@@ -1 +1 @@
-18.17.0
+18.20.0

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.17.0-bookworm
+FROM node:18.20.0-bullseye
 
 VOLUME /root/.yarn
 

README.md

@@ -77,9 +77,13 @@ This is a NodeJS backend that manages draft Nodes. It maintains a user auth, ver
 
 ## **nodes-lib**
 
-A library for programmatically interacting with Nodes, basically allowing automation of actions possible in the webapp. See separate documentation in the [README](./nodes-lib/README.md)
+A library for programmatically interacting with Nodes, basically allowing automation of actions possible in the webapp. See separate documentation in the [README](./nodes-lib/README.md).
 <br><br>
 
+## **reverse-proxy-service**
+
+A tiny service for proxying route segments to given target URL's, allowing hiding many target destinations under aliases on one domain. See docs in [README](./reverse-proxy-service/README.md).
+
 ## **desci-art-viewer**
 
 Nobody knows why this is still here, but it implements a React+Three.js 3d torus that plays [Conway's Game of Life](https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life) on the surface of the torus. We were totally inspired by [this gif on Wikipedia](https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life#/media/File:Trefoil_knot_conways_game_of_life.gif) and it only seems to work on Mac/Linux right now, YMMV.

bootstrapCeramic.sh

@@ -10,12 +10,17 @@
 # There is no damage trying to run this multiple times in a row; it's
 # idempotent.
 
+set -euo pipefail
+
 CTX="[bootstrapCeramic.sh]"
+WAS_RUNNING=0
 
-set -euo pipefail
 trap catch ERR
 catch() {
-  echo "$CTX script failed (are CODEX_REPO_PATH and TOGGLE_CERAMIC set in .env?)"
+  echo "$CTX script failed! Have you set CERAMIC_ADMIN_SEED in .env?"
+  if [ "$WAS_RUNNING" -eq "0" ]; then
+    docker compose --project-name desci down
+  fi
   exit 1
 }
 
@@ -25,38 +30,48 @@ if [[ ! -f .env ]]; then
   exit 1
 fi
 
-# Assert desci-codex repo available
-CODEX_REPO_PATH=$(grep "CODEX_REPO_PATH" .env | cut -d"=" -f2)
-if [[ -z "$CODEX_REPO_PATH" ]]; then
-  echo "$CTX CODEX_REPO_PATH not set in .env, aborting!"
+# Make sure we have the admin seed in env so modelIDs make sense
+CERAMIC_ADMIN_SEED=$(grep "CERAMIC_ADMIN_SEED" .env | cut -d"=" -f2)
+if [[ -z "$CERAMIC_ADMIN_SEED" ]]; then
+  echo "$CTX CERAMIC_ADMIN_SEED must be set in env, as the modelID's aren't deterministic otherwise."
   exit 1
-else
-  echo "$CTX Found codex repo path: $CODEX_REPO_PATH"
 fi
 
-# Assert ceramic service is running
+# Check if ceramic service is already running
+WAS_RUNNING=0
 RUNNING_SERVICES=$(docker compose --project-name desci ps --services)
 if ! grep -q ceramic <<<"$RUNNING_SERVICES"; then
-  echo "$CTX the ceramic compose service doesn't seem to be running, aborting!"
-  exit 1
+  echo "$CTX the ceramic compose service doesn't seem to be running, starting..."
+  docker compose \
+    -f docker-compose.dev.yml \
+    -f docker-compose.yml \
+    --project-name desci \
+    up ceramic \
+    --detach
+  sleep 5
+else
+  echo "$CTX Ceramic service already running, won't touch compose services..."
+  WAS_RUNNING=1
 fi
 
-# Setup desci-codex and deploy composites
-pushd "$CODEX_REPO_PATH"
+echo "$CTX Downloading the runtime definition file for the composeDB models..."
+curl -L --output .composedbRuntimeDefinition.json \
+  https://raw.githubusercontent.com/desci-labs/desci-codex/main/packages/composedb/src/__generated__/definition.json
 
-# Check that the node admin secret is set up, otherwise the model ID's wont be correct
-if [ ! -f "packages/composedb/admin_seed.txt" ]; then
-  echo "$CTX Composites need to be deployed with the ceramic node admin seed for the local node, as the model IDs aren't deterministic otherwise"
-  exit 1
-fi
+echo "$CTX Deploying composites to ceramic node..."
+npx --yes @composedb/cli composite:deploy \
+  .composedbRuntimeDefinition.json \
+  --ceramic-url="http://localhost:7007" \
+  --did-private-key="$CERAMIC_ADMIN_SEED"
 
-if [[ ! -d "node_modules" ]]; then
-  echo "$CTX installing deps desci-codex..."
-  npm ci
-fi
+sleep 5
+echo "$CTX Deployment all good, probably!"
 
-echo "$CTX deploying composites..."
-npm run --workspace packages/composedb deployComposites
-popd
+if [ "$WAS_RUNNING" -eq "0" ]; then
+  echo "$CTX Shutting down ceramic service..."
+  docker compose --project-name desci down
+else
+  echo "$CTX Leaving compose services up as they were already running when we started."
+fi 
 
-echo "$CTX Done! Re-run this script if local state is cleaned."
+echo "$CTX Done! You need to run me again if local data is wiped."

ceramic-k8s/ceramic_deployment_dev.yaml

@@ -48,26 +48,19 @@ spec:
               cpu: "0.6"
               memory: 4Gi
           # # restart pod after failureThreshold*periodSeconds total seconds
-          # livenessProbe:
-          #   httpGet:
-          #     path: /
-          #     port: server-api
-          #   failureThreshold: 80
-          #   periodSeconds: 3
-          # # temporarily stop sending traffic to pod after failureThreshold*periodSeconds total seconds
-          # readinessProbe:
-          #   httpGet:
-          #     path: /
-          #     port: server-api
-          #   failureThreshold: 3
-          #   periodSeconds: 1
+          livenessProbe:
+            httpGet:
+              path: /api/v0/node/healthcheck
+              port: http-api
+            failureThreshold: 10
+            periodSeconds: 5
           # # wait for pod to start for failureThreshold*periodSeconds total seconds
-          # startupProbe:
-          #   httpGet:
-          #     path: /
-          #     port: server-api
-          #   failureThreshold: 200
-          #   periodSeconds: 1
+          startupProbe:
+            httpGet:
+              path: /api/v0/node/healthcheck
+              port: http-api
+            failureThreshold: 60
+            periodSeconds: 1
 
       serviceAccountName: "vault-auth"
 ---