Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
hubsmoke committed Jan 30, 2024
2 parents 7d79aed + 9d3523f commit f218b9f
Show file tree
Hide file tree
Showing 261 changed files with 19,003 additions and 5,388 deletions.
35 changes: 35 additions & 0 deletions .ceramicDev.config.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{
"anchor": { },
"http-api": {
"cors-allowed-origins": [
".*"
],
"admin-dids": [
"did:key:z6MktbKJrMnhVJ37QFTo12911ycm2juKDUzWHDVETu9s5a9T"
]
},
"ipfs": {
"mode": "remote",
"host": "http://host.docker.internal:5001"
},
"logger": {
"log-level": 2
},
"metrics": {
"metrics-exporter-enabled": false,
"metrics-port": 9090
},
"network": {
"name": "inmemory"
},
"node": {},
"state-store": {
"mode": "fs",
"local-directory": "/root/.ceramic/statestore"
},
"indexing": {
"db": "postgres://walter:white@db_postgres:5432/postgres",
"allow-queries-before-historical-sync": true,
"models": []
}
}
35 changes: 35 additions & 0 deletions .ceramicTemplate.config.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{
"anchor": { },
"http-api": {
"cors-allowed-origins": [
".*"
],
"admin-dids": [
"did:key:z6MktbKJrMnhVJ37QFTo12911ycm2juKDUzWHDVETu9s5a9T"
]
},
"ipfs": {
"mode": "remote",
"host": "@DOCKER_HOST@"
},
"logger": {
"log-level": 2
},
"metrics": {
"metrics-exporter-enabled": false,
"metrics-port": 9090
},
"network": {
"name": "inmemory"
},
"node": {},
"state-store": {
"mode": "fs",
"local-directory": "/root/.ceramic/statestore"
},
"indexing": {
"db": "@POSTGRES_URI@",
"allow-queries-before-historical-sync": true,
"models": []
}
}
15 changes: 11 additions & 4 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ PORT=5420
IPFS_NODE_URL=http://host.docker.internal:5001
PUBLIC_IPFS_RESOLVER=https://ipfs.io

# IPFS_RESOLVER_OVERRIDE=http://host.docker.internal:8089/ipfs

### Database - Postgres
PG_HOST=db_boilerplate
PG_PORT=5432
Expand All @@ -29,7 +31,7 @@ REDIS_URL=redis://host.docker.internal:6379

# LOCAL DEV
MNEMONIC=test test test test test test test test test test test junk
# http://localhost:1984
# http://localhost:1984
#
# LIVE:
# host: 'arweave.net',
Expand Down Expand Up @@ -75,8 +77,6 @@ VSCODE_ACCESS_TOKEN=
# nodes media server (DOI resolution, video transcoding, LaTeX -> PDF Conversion)
NODES_MEDIA_SERVER_URL=http://host.docker.internal:5454

DISABLE_EXTERNAL_PUBLISH=1

# honeycomb.io telemetry
OTEL_SERVICE_NAME=
HONEYCOMB_API_KEY=
Expand All @@ -88,4 +88,11 @@ MEDIA_SECRET_KEY=supersecret

ORCID_API_DOMAIN=https://api.sandbox.orcid.org
ORCID_CLIENT_ID=
ORCID_CLIENT_SECRET=
ORCID_CLIENT_SECRET=

REPO_SERVER_URL=http://host.docker.internal:5445
REPO_SERVICE_SECRET_KEY=secretrepo
# Ceramic publish feature toggle, set to 1 for active
TOGGLE_CERAMIC=
# If above is set, clone `@desci-labs/desci-codex` and put the path to it here
CODEX_REPO_PATH=
5 changes: 4 additions & 1 deletion .env.test
Original file line number Diff line number Diff line change
Expand Up @@ -69,4 +69,7 @@ CSO_CLASSIFIER_API=
VSCODE_ACCESS_TOKEN=

# nodes media server (DOI resolution, video transcoding, LaTeX -> PDF Conversion)
NODES_MEDIA_SERVER_URL=http://host.docker.internal:5454
NODES_MEDIA_SERVER_URL=http://host.docker.internal:5454

REPO_SERVICE_SECRET_KEY="m8sIy5BPygBcX3+ZmMVuAA10k6w59BSCZd+Z5+VLYm4="
REPO_SERVER_URL=http://host.docker.internal:5485
1 change: 0 additions & 1 deletion .github/PULL_REQUEST_TEMPLATE.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
Asana ticket: <ticket_link>
Closes #<GH_issue_number>

## Description of the Problem / Feature
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/build-and-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ on:
paths:
- .github/workflows/**
- desci-server/**
- desci-repo/**
- desci-contracts/**
- Dockerfile

Expand Down Expand Up @@ -47,7 +48,7 @@ jobs:
docker info
- name: Install dependencies
run: cd desci-models && npm i -g yarn && yarn && yarn build && cd ../desci-server && yarn
run: cd desci-models && npm i -g yarn && yarn && yarn build && cd ../desci-server && yarn && cd ../desci-repo && yarn

- name: Stub contract
run: |
Expand Down
187 changes: 187 additions & 0 deletions .github/workflows/build-repo-server.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
# build.yml
on:
# pull_request:
# paths:
# - desci-server/**
push:
paths:
- .github/workflows/**
- desci-repo/**
branches: # array of glob patterns matching against refs/heads. Optional; defaults to all
- main # triggers on pushes that contain changes
- develop
- demo

name: Build desci-repo

# https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
env:
AWS_DEFAULT_REGION: us-east-2
AWS_DEFAULT_OUTPUT: json
AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
CONTAINER_IMAGE: desci-repo-server
DOCKER_BUILDKIT: 1

jobs:
# build-and-test:
# runs-on: ubuntu-latest
# # container:
# # image: ubuntu:latest
# # options: --user 1001

# steps:
# - name: Check out repository
# uses: actions/checkout@v2

# - name: Set up the environment
# # Replace this line with the appropriate setup for your project
# # Examples:
# uses: actions/setup-node@v2
# with:
# node-version: 16

# - name: Set up docker-compose
# run: |
# sudo curl -L "https://github.com/docker/compose/releases/download/v2.18.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
# sudo docker-compose --version
# docker info

# - name: Install dependencies
# run: cd desci-repo && yarn

# - name: Run tests
# run: |
# cd desci-repo && export DOCKER_BUILDKIT=1 && yarn && yarn test
# echo "exit code $?"
# if [ $? -ne 0 ]; then
# exit 1
# fi

build-and-push:
# needs: build-and-test
name: Build and deploy
runs-on: ubuntu-latest
steps:
- uses: hashicorp/setup-terraform@v1
- name: Checkout
uses: actions/checkout@master

# Add steps here like linting, testing, minification, etc.
- id: install-aws-cli
uses: unfor19/install-aws-cli-action@v1
with:
version: 1

- uses: prepor/action-aws-iam-authenticator@master
- run: aws-iam-authenticator version

- name: Install Kubectl
run: |
#$(curl -Ls https://dl.k8s.io/release/stable.txt)
version=v1.23.6
echo "using kubectl@$version"
curl -sLO "https://dl.k8s.io/release/$version/bin/linux/amd64/kubectl" -o kubectl
chmod +x kubectl
mv kubectl /usr/local/bin
mkdir $HOME/.kube
sudo apt-get update
sudo apt-get install less
echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
aws sts get-caller-identity
kubectl describe deployments
- name: Build and tag the image (DEV)
if: github.ref == 'refs/heads/develop'
run: |
# Build and tag the image
docker build \
-t $CONTAINER_IMAGE-dev:latest \
-t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev \
./desci-repo
- name: Build and tag the image (DEMO)
if: github.ref == 'refs/heads/demo'
run: |
# Build and tag the image
docker build \
-t $CONTAINER_IMAGE-demo:latest \
-t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo \
./desci-repo
- name: Build and tag the image (PROD)
if: github.ref == 'refs/heads/main'
run: |
# Build and tag the image
docker build \
-t $CONTAINER_IMAGE:latest \
-t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE \
./desci-repo
# Add additional steps here like scanning of image

# Only push to registry on dev
- name: Push (DEV)
if: github.ref == 'refs/heads/develop'
run: |
# Push image to AWS ECR
aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
docker tag $CONTAINER_IMAGE-dev:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
docker tag $CONTAINER_IMAGE-dev:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
- name: Push (DEMO)
if: github.ref == 'refs/heads/demo'
run: |
# Push image to AWS ECR
aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
docker tag $CONTAINER_IMAGE-demo:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }}
docker tag $CONTAINER_IMAGE-demo:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:latest
docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }}
docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:latest
- name: Push (PROD)
if: github.ref == 'refs/heads/main'
run: |
# Push image to AWS ECR
aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
docker tag $CONTAINER_IMAGE:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }}
docker tag $CONTAINER_IMAGE:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:latest
docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }}
docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:latest
- name: Deploy to EKS (DEV)
# uses: steebchen/kubectl@v2.0.0
if: github.ref == 'refs/heads/develop'
run: | # defaults to latest kubectl binary version
kubectl apply -f desci-repo/kubernetes/deployment_dev.yaml
kubectl set image deployment/desci-repo-server-dev desci-repo-server-dev=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }} --record
- name: Deploy to EKS (DEMO)
# uses: steebchen/kubectl@v2.0.0
if: github.ref == 'refs/heads/demo'
run: | # defaults to latest kubectl binary version
kubectl set image deployment/desci-repo-server-demo desci-repo-server-demo=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-demo:${{ github.sha }} --record
- name: Deploy to EKS (PROD)
if: github.ref == 'refs/heads/main'
run: | # defaults to latest kubectl binary version
kubectl apply -f desci-repo/kubernetes/deployment.yaml
kubectl set image deployment/desci-repo-server desci-repo-server=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }} --record
- name: Verify EKS Deployment (DEV)
if: github.ref == 'refs/heads/develop'
run: |
kubectl rollout status deployment/desci-repo-server-dev
- name: Verify EKS Deployment (DEMO)
if: github.ref == 'refs/heads/demo'
run: |
kubectl rollout status deployment/desci-repo-server-demo
- name: Verify EKS Deployment (PROD)
if: github.ref == 'refs/heads/main'
run: |
kubectl rollout status deployment/desci-repo-server
2 changes: 2 additions & 0 deletions .github/workflows/build-server.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@ jobs:
# uses: steebchen/kubectl@v2.0.0
if: github.ref == 'refs/heads/develop'
run: | # defaults to latest kubectl binary version
kubectl apply -f desci-server/kubernetes/deployment_dev.yaml
kubectl set image deployment/desci-server-dev desci-server-dev=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }} --record
- name: Deploy to EKS (DEMO)
Expand All @@ -190,6 +191,7 @@ jobs:
- name: Deploy to EKS (PROD)
if: github.ref == 'refs/heads/main'
run: | # defaults to latest kubectl binary version
kubectl apply -f desci-server/kubernetes/deployment.yaml
kubectl set image deployment/desci-server desci-server=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }} --record
- name: Verify EKS Deployment (DEV)
Expand Down
22 changes: 21 additions & 1 deletion .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,34 @@
"remoteRoot": "/app"
},
{
"name": "Docker: Attach to Desci Repo",
"type": "node",
"request": "attach",
"name": "Mocha Tests",
"restart": true,
"port": 9232,
"address": "localhost",
"localRoot": "${workspaceFolder}",
"remoteRoot": "/app"
},
{
"type": "node",
"request": "attach",
"name": "Docker: Mocha Tests",
"restart": true,
"port": 9227,
"address": "localhost",
"localRoot": "${workspaceFolder}",
"remoteRoot": "/app"
},
{
"type": "node",
"request": "attach",
"name": "Scripts Debug",
"restart": true,
"port": 9277,
"address": "localhost",
"localRoot": "${workspaceFolder}",
"remoteRoot": "/app"
}
]
}
Loading

0 comments on commit f218b9f

Please sign in to comment.