Merge pull request #230 from desci-labs/develop

promote main
desci-labs · Feb 26, 2024 · 7b779da · 7b779da
2 parents 7d21252 + e0bd39b
commit 7b779da
Show file tree

Hide file tree

Showing 53 changed files with 4,357 additions and 59 deletions.
diff --git a/.env.example b/.env.example
@@ -98,5 +98,11 @@ TOGGLE_CERAMIC=
 # If above is set, clone `@desci-labs/desci-codex` and put the path to it here
 CODEX_REPO_PATH=
 
+
+
+# ISOLATED MEDIA SERVER
+ISOLATED_MEDIA_SERVER_URL=http://media_isolated:7771
+IPFS_READ_ONLY_GATEWAY_SERVER=http://host.docker.internal:8089/ipfs # Used to proxy ipfs requests for ISOLATED_MEDIA_SERVER
+
 # SET TO 1 to run communities seed script
-RUN=1
+RUN=1
diff --git a/.github/workflows/build-isolated-media-server.yaml b/.github/workflows/build-isolated-media-server.yaml
@@ -0,0 +1,120 @@
+# build.yml
+on:
+  workflow_dispatch: # This line enables manual triggers
+  push:
+    paths:
+      - .github/workflows/**
+      - desci-media-isolated/**
+      - Dockerfile
+    branches: # array of glob patterns matching against refs/heads. Optional; defaults to all
+      - main # triggers on pushes that contain changes
+      - develop
+
+      # TODO: add demo env
+
+name: Build desci-isolated-media-server
+
+# https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
+env:
+  AWS_DEFAULT_REGION: us-east-2
+  AWS_DEFAULT_OUTPUT: json
+  AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  CONTAINER_IMAGE: desci-media-isolated
+  DOCKER_BUILDKIT: 1
+
+jobs:
+  build-and-push:
+    name: Build and deploy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: hashicorp/setup-terraform@v1
+      - name: Checkout
+        uses: actions/checkout@master
+
+      # Add steps here like linting, testing, minification, etc.
+      - id: install-aws-cli
+        uses: unfor19/install-aws-cli-action@v1
+        with:
+          version: 1
+
+      - uses: prepor/action-aws-iam-authenticator@master
+      - run: aws-iam-authenticator version
+
+      - name: Install Kubectl
+        run: |
+          #$(curl -Ls https://dl.k8s.io/release/stable.txt)
+          version=v1.23.6
+          echo "using kubectl@$version"
+          curl -sLO "https://dl.k8s.io/release/$version/bin/linux/amd64/kubectl" -o kubectl
+          chmod +x kubectl
+          mv kubectl /usr/local/bin
+          mkdir $HOME/.kube
+          sudo apt-get update
+          sudo apt-get install less
+          echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
+          aws sts get-caller-identity
+          kubectl describe deployments
+
+      - name: Build and tag the image (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          # Build and tag the image
+          docker build \
+            --target production -t $CONTAINER_IMAGE-dev:production \
+            --target production -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev \
+             ./desci-media-isolated
+
+      - name: Build and tag the image (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          # Build and tag the image
+          docker build \
+            --target production -t $CONTAINER_IMAGE-prod:production \
+            --target production -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE \
+             ./desci-media-isolated
+
+      # Add additional steps here like scanning of image
+
+      # Only push to registry on master
+      - name: Push (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          # Push image to AWS ECR
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker tag $CONTAINER_IMAGE-dev:production $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
+          docker tag $CONTAINER_IMAGE-dev:production $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:production
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:production
+
+      - name: Push (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          # Push image to AWS ECR
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker tag $CONTAINER_IMAGE:production $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:${{ github.sha }}
+          docker tag $CONTAINER_IMAGE:production $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:production
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:${{ github.sha }}
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:production
+
+      - name: Deploy to EKS (DEV)
+        # uses: steebchen/kubectl@v2.0.0
+        if: github.ref == 'refs/heads/develop'
+        run: | # defaults to latest kubectl binary version
+          kubectl set image deployment/desci-media-isolated-dev desci-media-isolated-dev=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }} --record
+
+      - name: Deploy to EKS (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: | # defaults to latest kubectl binary version
+          kubectl set image deployment/desci-media-isolated-prod desci-media-isolated-prod=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:${{ github.sha }} --record
+
+      - name: Verify EKS Deployment (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          kubectl rollout status deployment/desci-media-isolated-dev
+
+      - name: Verify EKS Deployment (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          kubectl rollout status deployment/desci-media-isolated-prod
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -43,6 +43,22 @@
       "address": "localhost",
       "localRoot": "${workspaceFolder}",
       "remoteRoot": "/app"
+    },
+    {
+      "name": "media-isolated tsx",
+      "type": "node",
+      "request": "attach",
+      "restart": true,
+      "localRoot": "${workspaceFolder}/desci-media-isolated",
+      "remoteRoot": "/usr/src/app",
+      "port": 9777,
+      "skipFiles": [
+        // Node.js internal core modules
+        "<node_internals>/**",
+
+        // Ignore all dependencies (optional)
+        "${workspaceFolder}/node_modules/**"
+      ]
     }
   ]
 }
diff --git a/Makefile b/Makefile
@@ -42,3 +42,6 @@ nodes-media/.env:
 desci-repo/.env:
 	if [ ! -f desci-repo/.env ]; then cp desci-repo/.env.example desci-repo/.env; fi
 
+desci-media-isolated/.env:
+	if [ ! -f desci-media-isolated/.env ]; then cp desci-media-isolated/.env.example desci-media-isolated/.env; fi
+
diff --git a/desci-media-isolated/.dockerignore b/desci-media-isolated/.dockerignore
@@ -0,0 +1,6 @@
+config
+database
+dist
+node_modules
+.git
+# .env
diff --git a/desci-media-isolated/.env.example b/desci-media-isolated/.env.example
@@ -0,0 +1,5 @@
+NODE_ENV=development
+
+PORT=7771
+
+IPFS_GATEWAY=http://host.docker.internal:5420/v1/ipfs
diff --git a/desci-media-isolated/.eslintrc.json b/desci-media-isolated/.eslintrc.json
@@ -0,0 +1,17 @@
+{
+  "env": {
+    "browser": true,
+    "es2021": true
+  },
+  "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "ecmaVersion": "latest",
+    "sourceType": "module"
+  },
+  "plugins": ["@typescript-eslint"],
+  "rules": {
+    "@typescript-eslint/no-explicit-any": 0,
+    "@typescript-eslint/no-unused-vars": 0
+  }
+}
diff --git a/desci-media-isolated/.gitignore b/desci-media-isolated/.gitignore
@@ -0,0 +1,5 @@
+.env
+node_modules
+dist
+.npm
+.temp
diff --git a/desci-media-isolated/.nvmrc b/desci-media-isolated/.nvmrc
@@ -0,0 +1 @@
+20.9.0
diff --git a/desci-media-isolated/.prettierignore b/desci-media-isolated/.prettierignore
@@ -0,0 +1 @@
+dist
diff --git a/desci-media-isolated/.prettierrc b/desci-media-isolated/.prettierrc
@@ -0,0 +1,6 @@
+{
+  "printWidth": 120,
+  "semi": true,
+  "singleQuote": true,
+  "trailingComma": "all"
+}
diff --git a/desci-media-isolated/Dockerfile b/desci-media-isolated/Dockerfile
@@ -0,0 +1,54 @@
+
+FROM docker.io/node:20.9.0 as base
+
+# Install dumb-init so we can use it as PID 1
+
+RUN apt-get update && apt-get install -y dumb-init ghostscript unoconv ffmpeg imagemagick curl && \
+    rm -rf /var/lib/apt/lists/*
+
+# Modify ImageMagick policy to allow PDF processing
+RUN sed -i '/<policy domain="coder" rights="none" pattern="PDF" \/>/c\<policy domain="Undefined" rights="read|write" pattern="PDF" \/>' /etc/ImageMagick-6/policy.xml
+
+
+# App Setup
+WORKDIR /usr/src/app
+
+COPY tsconfig.json .
+COPY package*.json ./
+
+FROM base as dev
+
+RUN --mount=type=cache,target=/usr/src/app/.npm \
+  npm set cache /usr/src/app/.npm && \
+  npm install
+
+
+COPY . .
+
+# Expose debugger port
+EXPOSE 9777
+
+ENTRYPOINT ["/usr/src/app/scripts/containerInitDev.sh"]
+CMD ["dumb-init", "npx", "tsx","watch", "--env-file=.env", "--inspect=0.0.0.0:9777", "src/index.ts"]
+
+FROM base as production
+# Cache mounts for faster builds, prod env for better express perf
+RUN --mount=type=cache,target=/usr/src/app/.npm \
+  npm set cache /usr/src/app/.npm && \
+  npm install
+
+ENV NODE_ENV production
+# 'node' user is created by the node image, prevent perm issues, run with reduced privs
+RUN mkdir -p /usr/src/app/dist && chown node:node /usr/src/app/dist
+USER node
+COPY --chown=node:node ./src/ ./src/
+USER root
+RUN chown -R node:node /usr/src/app
+USER node
+RUN npm run build && \
+    npm prune --production
+
+
+COPY --chown=node:node ./scripts/containerInitProd.sh /usr/src/app/scripts/containerInitProd.sh
+ENTRYPOINT ["/usr/src/app/scripts/containerInitProd.sh"]
+CMD ["dumb-init", "node", "dist/index.js"]
diff --git a/desci-media-isolated/kubernetes/deployment.yaml b/desci-media-isolated/kubernetes/deployment.yaml
@@ -0,0 +1,56 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: desci-media-isolated-prod-service
+  labels:
+    App: DesciMediaIsolatedProd
+spec:
+  type: ClusterIP
+  selector:
+    App: DesciMediaIsolatedProd
+  ports:
+    - name: api
+      port: 7771
+      targetPort: 7771
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: desci-media-isolated-prod
+  labels:
+    App: DesciMediaIsolatedProd
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      App: DesciMediaIsolatedProd
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        App: DesciMediaIsolated
+    spec:
+      containers:
+        - image: 523044037273.dkr.ecr.us-east-2.amazonaws.com/desci-media-isolated-prod:production
+          name: desci-media-isolated-prod
+          ports:
+            - containerPort: 7771
+              name: api
+          env:
+            - name: PORT
+              value: '7771'
+            - name: IPFS_GATEWAY
+              value: 'http://host.docker.internal:5420/v1/ipfs'
+          resources:
+            limits:
+              cpu: '0.5'
+              memory: 2Gi
+            requests:
+              cpu: 250m
+              memory: 1Gi
+      serviceAccountName: 'default'
diff --git a/desci-media-isolated/kubernetes/deployment_dev.yaml b/desci-media-isolated/kubernetes/deployment_dev.yaml
@@ -0,0 +1,56 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: desci-media-isolated-dev-service
+  labels:
+    App: DesciMediaIsolatedDev
+spec:
+  type: ClusterIP
+  selector:
+    App: DesciMediaIsolatedDev
+  ports:
+    - name: api
+      port: 7771
+      targetPort: 7771
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: desci-media-isolated-dev
+  labels:
+    App: DesciMediaIsolatedDev
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      App: DesciMediaIsolatedDev
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        App: DesciMediaIsolatedDev
+    spec:
+      containers:
+        - image: 523044037273.dkr.ecr.us-east-2.amazonaws.com/desci-media-isolated-dev:production
+          name: desci-media-isolated-dev
+          ports:
+            - containerPort: 7771
+              name: api
+          env:
+            - name: PORT
+              value: '7771'
+            - name: IPFS_GATEWAY
+              value: 'https://ipfs.desci.com/ipfs'
+          resources:
+            limits:
+              cpu: '0.5'
+              memory: 2Gi
+            requests:
+              cpu: 250m
+              memory: 1Gi
+      serviceAccountName: 'default'