Merge pull request #714 from desci-labs/tay/automerge-party-flagged

Automerge party Enabled (Stable)
desci-labs · Dec 17, 2024 · 4bba5c2 · 4bba5c2
2 parents 6d60623 + d732ac9
commit 4bba5c2
Show file tree

Hide file tree

Showing 74 changed files with 8,623 additions and 269 deletions.
diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml
@@ -9,6 +9,7 @@ on:
       - desci-server/**
       - desci-repo/**
       - desci-contracts/**
+      - sync-server/**
       - Dockerfile
 
 name: Test desci-server
@@ -32,6 +33,12 @@ jobs:
     #   options: --user 1001
 
     steps:
+      - name: Delete huge unnecessary tools folder
+        run: |
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
+          rm -rf /opt/hostedtoolcache
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
+
       - name: Check out repository
         uses: actions/checkout@v4
 
@@ -49,12 +56,15 @@ jobs:
             desci-repo/yarn.lock
 
       - name: Install dependencies
-        run: cd desci-models && npm i -g yarn && yarn && yarn build && cd ../desci-server && yarn --ignore-engines && cd ../desci-repo && yarn
+        run: |
+          cd desci-models && npm i -g yarn && yarn && yarn build && cd ../desci-server && yarn --ignore-engines && cd ../desci-repo && yarn
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
 
       - name: Stub contract
         run: |
           echo "{\"proxies\":[{\"address\":\"\"}]}" > desci-server/src/desci-contracts-config/unknown-research-object.json
           echo "{\"proxies\":[{\"address\":\"\"}]}" > desci-server/src/desci-contracts-config/unknown-dpid.json
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
 
       - name: Set up Sentry CLI
         run: |
@@ -65,9 +75,19 @@ jobs:
           echo -e "\nSENTRY_AUTH_TOKEN=$SENTRY_AUTH" >> desci-server/.env
           cd desci-server && yarn build
 
+      - name: Set up sync server
+        run: |
+          cd sync-server && yarn --ignore-engines && ./scripts/build.sh test
+          if [ $? -ne 0 ]; then
+            exit 1
+          fi
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
+
       - name: Run tests
         run: |
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
           cd desci-server && export DOCKER_BUILDKIT=1 && yarn --ignore-engines && yarn test
           if [ $? -ne 0 ]; then
             exit 1
           fi
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
diff --git a/.github/workflows/deploy-sync-server.yaml b/.github/workflows/deploy-sync-server.yaml
@@ -0,0 +1,48 @@
+name: Deploy Worker
+on:
+  pull_request:
+    paths:
+      - .github/workflows/deploy-sync-server.yaml
+      - sync-server/**
+    branches: # array of glob patterns matching against refs/heads. Optional; defaults to all
+      - main # triggers on pushes that contain changes
+      - develop
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      # - name: Show github ref
+      #   run: |
+      #     echo ${{ github.ref }}
+      #     echo ${{ github.ref_name }}
+      #     echo ${{ github.head_ref }}
+      # - name: Build & Deploy Worker (Dev)
+      #   if: github.head_ref == 'tay/automerge-party-flagged'
+      #   uses: cloudflare/wrangler-action@v3
+      #   with:
+      #     apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+      #     accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+      #     environment: staging
+      #     workingDirectory: sync-server
+      #     wranglerVersion: 3.95.0
+      - name: Build & Deploy Worker (Staging)
+        if: github.ref == 'refs/heads/develop'
+        uses: cloudflare/wrangler-action@v3
+        with:
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          environment: staging
+          workingDirectory: sync-server
+          wranglerVersion: 3.95.0
+      - name: Build & Deploy Worker (Production)
+        if: github.ref == 'refs/heads/main'
+        uses: cloudflare/wrangler-action@v3
+        with:
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          environment: production
+          workingDirectory: sync-server
+          wranglerVersion: 3.95.0
diff --git a/.gitignore b/.gitignore
@@ -18,4 +18,3 @@ local-data/
 **/*.iml
 .composedbRuntimeDefinition.json
 openalex-importer
-sync-server
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -43,5 +43,14 @@
   },
   "[sql]": {
     "editor.defaultFormatter": "ms-mssql.mssql"
+  },
+  "[dotenv]": {
+    "editor.defaultFormatter": "foxundermoon.shell-format"
+  },
+  "[dockerfile]": {
+    "editor.defaultFormatter": "ms-azuretools.vscode-docker"
+  },
+  "[shellscript]": {
+    "editor.defaultFormatter": "foxundermoon.shell-format"
   }
 }
diff --git a/Makefile b/Makefile
@@ -4,6 +4,7 @@ build: .env desci-contracts/.env
 	$(MAKE) -C desci-contracts build
 	$(MAKE) -C desci-server install
 	$(MAKE) -C desci-repo install
+	$(MAKE) -C sync-server install
 	$(MAKE) -C nodes-lib build
 
 .PHONY: sterile

diff --git a/desci-repo/.env.example b/desci-repo/.env.example
@@ -3,4 +3,9 @@ NODE_ENV=dev
 JWT_SECRET=secretshhh
 DATABASE_URL=postgresql://walter:white@db_boilerplate:5432/boilerplate
 IPFS_RESOLVER_OVERRIDE=http://host.docker.internal:8089/ipfs
-# DESCI_SERVER_URL=
+# DESCI_SERVER_URL=
+
+WRANGLER_HYPERDRIVE_LOCAL_CONNECTION_STRING_NODES_DB="postgresql://walter:white@host.docker.internal:5433/boilerplate"
+PARTY_SERVER_HOST=ws://localhost:5445
+PARTY_SERVER_TOKEN=auth-token
+ENABLE_PARTYKIT_FEATURE=true
diff --git a/desci-repo/.env.test b/desci-repo/.env.test
@@ -1,7 +1,7 @@
 NODE_ENV=test
 
 # these are mapped to different ports in docker-compose.test.yml
-PORT=5484 # mapped to 5486
+PORT=5484    # mapped to 5486
 WS_PORT=5445 # mapped to 5446
 
 IPFS_NODE_URL=http://host.docker.internal:5003
@@ -21,4 +21,8 @@ JWT_EXPIRATION=
 # FOR Communicating with REPO SERVICE
 REPO_SERVICE_SECRET_KEY="m8sIy5BPygBcX3+ZmMVuAA10k6w59BSCZd+Z5+VLYm4="
 
-IPFS_RESOLVER_OVERRIDE=http://host.docker.internal:8089/ipfs
+IPFS_RESOLVER_OVERRIDE=http://host.docker.internal:8089/ipfs
+
+PARTY_SERVER_HOST=host.docker.internal:5446
+PARTY_SERVER_TOKEN=auth-token
+ENABLE_PARTYKIT_FEATURE=true
diff --git a/desci-repo/Dockerfile b/desci-repo/Dockerfile
@@ -4,6 +4,8 @@ VOLUME /root/.yarn
 
 RUN npm install -g npm@9.8.1
 
+RUN apt-get update && apt-get install -y curl
+
 RUN mkdir /app
 WORKDIR /app
 RUN mkdir /app/repo-tmp
@@ -19,6 +21,6 @@ RUN yarn build
 EXPOSE 5484
 
 # websocket
-EXPOSE 5445
+# EXPOSE 5445
 
 CMD [ "yarn", "start" ]
diff --git a/desci-repo/kubernetes/deployment_dev.yaml b/desci-repo/kubernetes/deployment_dev.yaml
@@ -49,6 +49,9 @@ spec:
           export DATABASE_URL={{ .Data.DATABASE_URL }}
           export IPFS_RESOLVER_OVERRIDE={{ .Data.IPFS_RESOLVER_OVERRIDE }}
           export DESCI_SERVER_URL={{ .Data.DESCI_SERVER_URL }}
+          export PARTY_SERVER_HOST=nodes-dev-sync.desci.com
+          export PARTY_SERVER_TOKEN=auth-token
+          export ENABLE_PARTYKIT_FEATURE=true
           export PINO_LOG_LEVEL=info
           {{- end -}}
       labels:
@@ -63,8 +66,8 @@ spec:
           ports:
             - containerPort: 5484
               name: server-api
-            - containerPort: 5445
-              name: ws-api
+            # - containerPort: 5445
+            #   name: ws-api
           resources:
             limits:
               cpu: '4.0'

diff --git a/desci-repo/kubernetes/deployment_prod.yaml b/desci-repo/kubernetes/deployment_prod.yaml
@@ -50,6 +50,9 @@ spec:
           export DATABASE_URL={{ .Data.DATABASE_URL }}
           export IPFS_RESOLVER_OVERRIDE={{ .Data.IPFS_RESOLVER_OVERRIDE }}
           export DESCI_SERVER_URL={{ .Data.DESCI_SERVER_URL }}
+          export PARTY_SERVER_HOST=nodes-sync.desci.com
+          export PARTY_SERVER_TOKEN=auth-token
+          export ENABLE_PARTYKIT_FEATURE=true
           export PINO_LOG_LEVEL=info
           {{- end -}}
       labels:
@@ -64,8 +67,8 @@ spec:
           ports:
             - containerPort: 5484
               name: server-api
-            - containerPort: 5445
-              name: ws-api
+            # - containerPort: 5445
+            #   name: ws-api
           resources:
             limits:
               cpu: '4'

diff --git a/desci-repo/kubernetes/deployment_staging.yaml b/desci-repo/kubernetes/deployment_staging.yaml
@@ -50,6 +50,9 @@ spec:
           export DATABASE_URL={{ .Data.DATABASE_URL }}
           export IPFS_RESOLVER_OVERRIDE={{ .Data.IPFS_RESOLVER_OVERRIDE }}
           export DESCI_SERVER_URL={{ .Data.DESCI_SERVER_URL }}
+          export PARTY_SERVER_HOST=nodes-sync.desci.com
+          export PARTY_SERVER_TOKEN=auth-token
+          export ENABLE_PARTYKIT_FEATURE=true
           export PINO_LOG_LEVEL=info
           {{- end -}}
       labels:
@@ -64,8 +67,8 @@ spec:
           ports:
             - containerPort: 5484
               name: server-api
-            - containerPort: 5445
-              name: ws-api
+            # - containerPort: 5445
+            #   name: ws-api
           resources:
             limits:
               cpu: '2'

diff --git a/desci-repo/package.json b/desci-repo/package.json
@@ -4,12 +4,13 @@
   "description": "",
   "type": "module",
   "engines": {
-    "node": ">=16.14"
+    "node": ">=16"
   },
   "scripts": {
     "build": "rimraf dist && tsc",
     "start": "NODE_PATH=./dist node ./dist/index.js",
     "dev": "npm-run-all --parallel watch-compile watch-dev",
+    "partykit:dev": "wrangler dev --port 5446",
     "docker:dev": "docker compose --file docker-compose.repo.yml --compatibility up --build",
     "watch-dev": "debug=* nodemon --watch \"dist/**/*\" -e js ./dist/index.js",
     "watch-compile": "debug=* tsc -w --preserveWatchOutput",
@@ -35,10 +36,6 @@
     "cliui": "^8.0.1"
   },
   "devDependencies": {
-    "@automerge/automerge": "^2.1.10",
-    "@automerge/automerge-repo": "^1.1.0",
-    "@automerge/automerge-repo-network-websocket": "^1.0.19",
-    "@automerge/automerge-repo-storage-nodefs": "^1.0.19",
     "@types/chai": "^4.3.11",
     "@types/cors": "^2.8.17",
     "@types/deep-equal": "^1.0.4",
@@ -71,6 +68,10 @@
     "typescript": "5.1.6"
   },
   "dependencies": {
+    "@automerge/automerge-repo": "^1.2.1",
+    "@automerge/automerge-repo-network-websocket": "^1.2.1",
+    "@automerge/automerge-repo-storage-indexeddb": "^1.2.1",
+    "@cloudflare/workers-types": "^4.20241022.0",
     "@desci-labs/desci-models": "^0.2.19",
     "@sentry/node": "8.29.0",
     "@sentry/profiling-node": "8.32.0",
@@ -80,14 +81,20 @@
     "deep-equal": "^2.2.3",
     "dotenv": "^16.3.1",
     "express": "^4.18.2",
+    "isomorphic-ws": "^5.0.0",
     "jsonwebtoken": "^9.0.2",
     "morgan": "^1.10.0",
-    "pg": "^8.11.3",
+    "partykit": "^0.0.111",
+    "partyserver": "^0.0.57",
+    "partysocket": "^1.0.2",
+    "pg": "^8.13.1",
     "pino": "^8.16.2",
     "pino-http": "^8.5.1",
     "pino-pretty": "^10.2.3",
+    "postgres": "^3.4.5",
     "reflect-metadata": "^0.1.13",
     "uuid": "^9.0.1",
+    "wrangler": "^3.84.1",
     "ws": "^8.14.2",
     "zod": "^3.22.4"
   }

diff --git a/desci-repo/src/config.ts b/desci-repo/src/config.ts
@@ -4,3 +4,19 @@ export const PUBLIC_IPFS_PATH =
     : process.env.NODE_ENV === 'test'
       ? 'http://host.docker.internal:8091/ipfs'
       : 'https://ipfs.desci.com/ipfs';
+
+export const ENABLE_PARTYKIT_FEATURE =
+  process.env.ENABLE_PARTYKIT_FEATURE == '1' || process.env.ENABLE_PARTYKIT_FEATURE == 'true';
+export const PARTY_SERVER_HOST = process.env.PARTY_SERVER_HOST;
+export const PARTY_SERVER_TOKEN = process.env.PARTY_SERVER_TOKEN;
+
+export const IS_DEV = process.env.NODE_ENV == 'dev';
+export const IS_TEST = process.env.NODE_ENV == 'test';
+
+console.log({
+  env: process.env.ENABLE_PARTYKIT_FEATURE,
+  ENABLE_PARTYKIT_FEATURE,
+  PARTY_SERVER_HOST,
+  IS_DEV,
+  IS_TEST,
+});