Merge pull request #195 from desci-labs/repo-service

Repo service

.env.example

@@ -90,6 +90,8 @@ ORCID_API_DOMAIN=https://api.sandbox.orcid.org
 ORCID_CLIENT_ID=
 ORCID_CLIENT_SECRET=
 
+REPO_SERVER_URL=http://host.docker.internal:5445
+REPO_SERVICE_SECRET_KEY=secretrepo
 # Ceramic publish feature toggle, set to 1 for active
 TOGGLE_CERAMIC=
 # If above is set, clone `@desci-labs/desci-codex` and put the path to it here

.env.test

@@ -69,4 +69,7 @@ CSO_CLASSIFIER_API=
 VSCODE_ACCESS_TOKEN=
 
 # nodes media server (DOI resolution, video transcoding, LaTeX -> PDF Conversion)
-NODES_MEDIA_SERVER_URL=http://host.docker.internal:5454
+NODES_MEDIA_SERVER_URL=http://host.docker.internal:5454
+
+REPO_SERVICE_SECRET_KEY="m8sIy5BPygBcX3+ZmMVuAA10k6w59BSCZd+Z5+VLYm4="
+REPO_SERVER_URL=http://host.docker.internal:5485

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,3 @@
-Asana ticket: <ticket_link>
 Closes #<GH_issue_number>
 
 ## Description of the Problem / Feature

.github/workflows/build-and-test.yaml

@@ -7,6 +7,7 @@ on:
     paths:
       - .github/workflows/**
       - desci-server/**
+      - desci-repo/**
       - desci-contracts/**
       - Dockerfile
 
@@ -47,7 +48,7 @@ jobs:
           docker info
 
       - name: Install dependencies
-        run: cd desci-models && npm i -g yarn && yarn && yarn build && cd ../desci-server && yarn
+        run: cd desci-models && npm i -g yarn && yarn && yarn build && cd ../desci-server && yarn && cd ../desci-repo && yarn
 
       - name: Stub contract
         run: |

.github/workflows/build-repo-server.yaml

@@ -0,0 +1,187 @@
+# build.yml
+on:
+  # pull_request:
+  #   paths:
+  #   - desci-server/**
+  push:
+    paths:
+      - .github/workflows/**
+      - desci-repo/**
+    branches: # array of glob patterns matching against refs/heads. Optional; defaults to all
+      - main # triggers on pushes that contain changes
+      - develop
+      - demo
+
+name: Build desci-repo
+
+# https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
+env:
+  AWS_DEFAULT_REGION: us-east-2
+  AWS_DEFAULT_OUTPUT: json
+  AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  CONTAINER_IMAGE: desci-repo-server
+  DOCKER_BUILDKIT: 1
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    # container:
+    #   image: ubuntu:latest
+    #   options: --user 1001
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+
+      - name: Set up the environment
+        # Replace this line with the appropriate setup for your project
+        # Examples:
+        uses: actions/setup-node@v2
+        with:
+          node-version: 16
+
+      - name: Set up docker-compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/v2.18.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo docker-compose --version
+          docker info
+
+      - name: Install dependencies
+        run: cd desci-repo && yarn
+
+      - name: Run tests
+        run: |
+          cd desci-repo && export DOCKER_BUILDKIT=1 && yarn && yarn test
+          echo "exit code $?"
+          if [ $? -ne 0 ]; then
+            exit 1
+          fi
+
+  build-and-push:
+    needs: build-and-test
+    name: Build and deploy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: hashicorp/setup-terraform@v1
+      - name: Checkout
+        uses: actions/checkout@master
+
+      # Add steps here like linting, testing, minification, etc.
+      - id: install-aws-cli
+        uses: unfor19/install-aws-cli-action@v1
+        with:
+          version: 1
+
+      - uses: prepor/action-aws-iam-authenticator@master
+      - run: aws-iam-authenticator version
+
+      - name: Install Kubectl
+        run: |
+          #$(curl -Ls https://dl.k8s.io/release/stable.txt)
+          version=v1.23.6
+          echo "using kubectl@$version"
+          curl -sLO "https://dl.k8s.io/release/$version/bin/linux/amd64/kubectl" -o kubectl
+          chmod +x kubectl
+          mv kubectl /usr/local/bin
+          mkdir $HOME/.kube
+          sudo apt-get update
+          sudo apt-get install less
+          echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
+          aws sts get-caller-identity
+          kubectl describe deployments
+
+      - name: Build and tag the image (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          # Build and tag the image
+          docker build \
+            -t $CONTAINER_IMAGE-dev:latest \
+            -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev \
+             ./desci-repo
+
+      - name: Build and tag the image (DEMO)
+        if: github.ref == 'refs/heads/demo'
+        run: |
+          # Build and tag the image
+          docker build \
+            -t $CONTAINER_IMAGE-demo:latest \
+            -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo \
+             ./desci-repo
+
+      - name: Build and tag the image (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          # Build and tag the image
+          docker build \
+            -t $CONTAINER_IMAGE:latest \
+            -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE \
+             ./desci-repo
+
+      # Add additional steps here like scanning of image
+
+      # Only push to registry on dev
+      - name: Push (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          # Push image to AWS ECR
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker tag $CONTAINER_IMAGE-dev:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
+          docker tag $CONTAINER_IMAGE-dev:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
+
+      - name: Push (DEMO)
+        if: github.ref == 'refs/heads/demo'
+        run: |
+          # Push image to AWS ECR
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker tag $CONTAINER_IMAGE-demo:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }}
+          docker tag $CONTAINER_IMAGE-demo:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:latest
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }}
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:latest
+
+      - name: Push (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          # Push image to AWS ECR
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker tag $CONTAINER_IMAGE:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }}
+          docker tag $CONTAINER_IMAGE:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:latest
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }}
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:latest
+
+      - name: Deploy to EKS (DEV)
+        # uses: steebchen/kubectl@v2.0.0
+        if: github.ref == 'refs/heads/develop'
+        run: | # defaults to latest kubectl binary version
+          kubectl apply -f desci-repo/kubernetes/deployment_dev.yaml
+          kubectl set image deployment/desci-repo-server-dev desci-repo-server-dev=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }} --record
+
+      - name: Deploy to EKS (DEMO)
+        # uses: steebchen/kubectl@v2.0.0
+        if: github.ref == 'refs/heads/demo'
+        run: | # defaults to latest kubectl binary version
+          kubectl set image deployment/desci-repo-server-demo desci-repo-server-demo=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }} --record
+
+      - name: Deploy to EKS (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: | # defaults to latest kubectl binary version
+          kubectl apply -f desci-repo/kubernetes/deployment.yaml
+          kubectl set image deployment/desci-repo-server desci-repo-server=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }} --record
+
+      - name: Verify EKS Deployment (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          kubectl rollout status deployment/desci-repo-server-dev
+
+      - name: Verify EKS Deployment (DEMO)
+        if: github.ref == 'refs/heads/demo'
+        run: |
+          kubectl rollout status deployment/desci-repo-server-demo
+
+      - name: Verify EKS Deployment (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          kubectl rollout status deployment/desci-repo-server

.gitignore

@@ -15,5 +15,4 @@ ipfs-data/
 local-data/
 node-modules/
 node_modules/*
-.idea
-desci-repo
+.idea

.vscode/launch.json

@@ -14,6 +14,16 @@
       "localRoot": "${workspaceFolder}",
       "remoteRoot": "/app"
     },
+    {
+      "name": "Docker: Attach to Desci Repo",
+      "type": "node",
+      "request": "attach",
+      "restart": true,
+      "port": 9232,
+      "address": "localhost",
+      "localRoot": "${workspaceFolder}",
+      "remoteRoot": "/app"
+    },
     {
       "type": "node",
       "request": "attach",
@@ -22,7 +32,7 @@
       "port": 9227,
       "address": "localhost",
       "localRoot": "${workspaceFolder}",
-      "remoteRoot": "/app"
+      "remoteRoot": "/app",
     },
     {
       "type": "node",

Dockerfile

@@ -56,9 +56,6 @@ RUN yarn build
 # server api
 EXPOSE 5420
 
-# websocket
-EXPOSE 5445
-
 # prisma studio
 EXPOSE 5555
 

Makefile

@@ -25,7 +25,7 @@ clean-rec:
 	$(MAKE) -C desci-server clean
 
 .PHONY: .env
-.env: nodes-media/.env
+.env: nodes-media/.env desci-repo/.env
 	# Phony target, always runs but is idempotent
 	# Copies example env if not present, and fails until MNEMONIC is set
 	cp --no-clobber .env.example .env || true
@@ -37,3 +37,6 @@ desci-contracts/.env: .env
 nodes-media/.env:
 	cp nodes-media/.env.example nodes-media/.env
 
+desci-repo/.env:
+	cp desci-repo/.env.example desci-repo/.env
+

desci-repo/.dockerignore

@@ -0,0 +1,6 @@
+config
+database
+dist
+node_modules
+.git
+.env

desci-repo/.env.example

@@ -0,0 +1,5 @@
+REPO_SERVICE_SECRET_KEY=secretrepo
+NODE_ENV=dev
+JWT_SECRET=secretshhh
+DATABASE_URL=postgresql://walter:white@db_boilerplate:5432/boilerplate
+IPFS_RESOLVER_OVERRIDE=http://host.docker.internal:8089/ipfs

desci-repo/.env.test

@@ -0,0 +1,23 @@
+NODE_ENV=test
+
+PORT=5484
+
+IPFS_NODE_URL=http://host.docker.internal:5002
+PUBLIC_IPFS_RESOLVER=https://ipfs.io
+### Database - Postgres
+PG_HOST=host.docker.internal
+PG_PORT=5434
+POSTGRES_USER=walter
+POSTGRES_PASSWORD=white
+POSTGRES_DB=boilerplate
+DATABASE_URL=postgresql://walter:white@host.docker.internal:5434/boilerplate
+
+### JWT
+JWT_SECRET=secretshhh
+JWT_EXPIRATION=
+
+# FOR Communicating with REPO SERVICE
+REPO_SERVICE_SECRET_KEY="m8sIy5BPygBcX3+ZmMVuAA10k6w59BSCZd+Z5+VLYm4="
+REPO_SERVER_URL=http://host.docker.internal:5485
+
+IPFS_RESOLVER_OVERRIDE=http://host.docker.internal:8089/ipfs

desci-repo/.gitignore

@@ -0,0 +1,16 @@
+node_modules
+# Keep environment variables out of version control
+.env
+database/boilerplate/
+database/test/
+dist
+.nebulus
+log/access.log
+
+temp_downloads
+.nyc_output
+coverage
+
+server.log
+
+repo-tmp

desci-repo/.prettierignore

@@ -0,0 +1 @@
+dist

desci-repo/.prettierrc

@@ -0,0 +1,6 @@
+{
+  "printWidth": 120,
+  "semi": true,
+  "singleQuote": true,
+  "trailingComma": "all"
+}

desci-repo/Dockerfile

@@ -0,0 +1,21 @@
+FROM node:16.20.0-bookworm
+
+VOLUME /root/.yarn
+
+RUN npm install -g npm@8.10.0
+
+RUN mkdir /app
+WORKDIR /app
+
+COPY . .
+
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install
+RUN yarn build
+
+# server api
+EXPOSE 5484
+
+# websocket
+EXPOSE 5445
+
+CMD [ "yarn", "start" ]