Skip to content

Commit 97b9ee8

Browse files
TorantulinoTorantulino
authored
Merge branch 'dev' into master
2 parents 8352185 + e2df601 commit 97b9ee8

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+1466
-303
lines changed

‎.github/PULL_REQUEST_TEMPLATE.md

+13
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,16 @@ Here is a list of our critical paths, if you need some inspiration on what and h
2121
- Upload agent to marketplace
2222
- Import an agent from marketplace and confirm it executes correctly
2323
- Edit an agent from monitor, and confirm it executes correctly
24+
25+
### Configuration Changes 📝
26+
> [!NOTE]
27+
Only for the new autogpt platform, currently in autogpt_platform/
28+
29+
If you're making configuration or infrastructure changes, please remember to check you've updated the related infrastructure code in the autogpt_platform/infra folder.
30+
31+
Examples of such changes might include:
32+
33+
- Changing ports
34+
- Adding new services that need to communicate with each other
35+
- Secrets or environment variable changes
36+
- New or infrastructure changes such as databases

‎.github/workflows/platform-autgpt-deploy-prod.yml

+182
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
name: AutoGPT Platform - Build, Push, and Deploy Prod Environment
2+
3+
on:
4+
release:
5+
types: [published]
6+
7+
permissions:
8+
contents: 'read'
9+
id-token: 'write'
10+
11+
env:
12+
PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
13+
GKE_CLUSTER: prod-gke-cluster
14+
GKE_ZONE: us-central1-a
15+
NAMESPACE: prod-agpt
16+
17+
jobs:
18+
migrate:
19+
environment: production
20+
name: Run migrations for AutoGPT Platform
21+
runs-on: ubuntu-latest
22+
23+
steps:
24+
- name: Checkout code
25+
uses: actions/checkout@v2
26+
27+
- name: Set up Python
28+
uses: actions/setup-python@v4
29+
with:
30+
python-version: '3.11'
31+
32+
- name: Install Python dependencies
33+
run: |
34+
python -m pip install --upgrade pip
35+
pip install prisma
36+
37+
- name: Run Backend Migrations
38+
working-directory: ./autogpt_platform/backend
39+
run: |
40+
python -m prisma migrate deploy
41+
env:
42+
DATABASE_URL: ${{ secrets.BACKEND_DATABASE_URL }}
43+
44+
- name: Run Market Migrations
45+
working-directory: ./autogpt_platform/market
46+
run: |
47+
python -m prisma migrate deploy
48+
env:
49+
DATABASE_URL: ${{ secrets.MARKET_DATABASE_URL }}
50+
51+
build-push-deploy:
52+
environment: production
53+
name: Build, Push, and Deploy
54+
runs-on: ubuntu-latest
55+
56+
steps:
57+
- name: Checkout code
58+
uses: actions/checkout@v2
59+
with:
60+
fetch-depth: 0
61+
62+
- id: 'auth'
63+
uses: 'google-github-actions/auth@v1'
64+
with:
65+
workload_identity_provider: 'projects/638488734936/locations/global/workloadIdentityPools/prod-pool/providers/github'
66+
service_account: 'prod-github-actions-sa@agpt-prod.iam.gserviceaccount.com'
67+
token_format: 'access_token'
68+
create_credentials_file: true
69+
70+
- name: 'Set up Cloud SDK'
71+
uses: 'google-github-actions/setup-gcloud@v1'
72+
73+
- name: 'Configure Docker'
74+
run: |
75+
gcloud auth configure-docker us-east1-docker.pkg.dev
76+
77+
- name: Set up Docker Buildx
78+
uses: docker/setup-buildx-action@v1
79+
80+
- name: Cache Docker layers
81+
uses: actions/cache@v2
82+
with:
83+
path: /tmp/.buildx-cache
84+
key: ${{ runner.os }}-buildx-${{ github.sha }}
85+
restore-keys: |
86+
${{ runner.os }}-buildx-
87+
88+
- name: Check for changes
89+
id: check_changes
90+
run: |
91+
git fetch origin master
92+
BACKEND_CHANGED=$(git diff --name-only origin/master HEAD | grep "^autogpt_platform/backend/" && echo "true" || echo "false")
93+
FRONTEND_CHANGED=$(git diff --name-only origin/master HEAD | grep "^autogpt_platform/frontend/" && echo "true" || echo "false")
94+
MARKET_CHANGED=$(git diff --name-only origin/master HEAD | grep "^autogpt_platform/market/" && echo "true" || echo "false")
95+
echo "backend_changed=$BACKEND_CHANGED" >> $GITHUB_OUTPUT
96+
echo "frontend_changed=$FRONTEND_CHANGED" >> $GITHUB_OUTPUT
97+
echo "market_changed=$MARKET_CHANGED" >> $GITHUB_OUTPUT
98+
99+
- name: Get GKE credentials
100+
uses: 'google-github-actions/get-gke-credentials@v1'
101+
with:
102+
cluster_name: ${{ env.GKE_CLUSTER }}
103+
location: ${{ env.GKE_ZONE }}
104+
105+
- name: Build and Push Backend
106+
if: steps.check_changes.outputs.backend_changed == 'true'
107+
uses: docker/build-push-action@v2
108+
with:
109+
context: .
110+
file: ./autogpt_platform/backend/Dockerfile
111+
push: true
112+
tags: us-east1-docker.pkg.dev/agpt-prod/agpt-backend-prod/agpt-backend-prod:${{ github.sha }}
113+
cache-from: type=local,src=/tmp/.buildx-cache
114+
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
115+
116+
- name: Build and Push Frontend
117+
if: steps.check_changes.outputs.frontend_changed == 'true'
118+
uses: docker/build-push-action@v2
119+
with:
120+
context: .
121+
file: ./autogpt_platform/frontend/Dockerfile
122+
push: true
123+
tags: us-east1-docker.pkg.dev/agpt-prod/agpt-frontend-prod/agpt-frontend-prod:${{ github.sha }}
124+
cache-from: type=local,src=/tmp/.buildx-cache
125+
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
126+
127+
- name: Build and Push Market
128+
if: steps.check_changes.outputs.market_changed == 'true'
129+
uses: docker/build-push-action@v2
130+
with:
131+
context: .
132+
file: ./autogpt_platform/market/Dockerfile
133+
push: true
134+
tags: us-east1-docker.pkg.dev/agpt-prod/agpt-market-prod/agpt-market-prod:${{ github.sha }}
135+
cache-from: type=local,src=/tmp/.buildx-cache
136+
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
137+
138+
- name: Move cache
139+
run: |
140+
rm -rf /tmp/.buildx-cache
141+
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
142+
143+
- name: Set up Helm
144+
uses: azure/setup-helm@v1
145+
with:
146+
version: v3.4.0
147+
148+
- name: Deploy Backend
149+
if: steps.check_changes.outputs.backend_changed == 'true'
150+
run: |
151+
helm upgrade autogpt-server ./autogpt-server \
152+
--namespace ${{ env.NAMESPACE }} \
153+
-f autogpt-server/values.yaml \
154+
-f autogpt-server/values.prod.yaml \
155+
--set image.tag=${{ github.sha }}
156+
157+
- name: Deploy Websocket
158+
if: steps.check_changes.outputs.backend_changed == 'true'
159+
run: |
160+
helm upgrade autogpt-websocket-server ./autogpt-websocket-server \
161+
--namespace ${{ env.NAMESPACE }} \
162+
-f autogpt-websocket-server/values.yaml \
163+
-f autogpt-websocket-server/values.prod.yaml \
164+
--set image.tag=${{ github.sha }}
165+
166+
- name: Deploy Market
167+
if: steps.check_changes.outputs.market_changed == 'true'
168+
run: |
169+
helm upgrade autogpt-market ./autogpt-market \
170+
--namespace ${{ env.NAMESPACE }} \
171+
-f autogpt-market/values.yaml \
172+
-f autogpt-market/values.prod.yaml \
173+
--set image.tag=${{ github.sha }}
174+
175+
- name: Deploy Frontend
176+
if: steps.check_changes.outputs.frontend_changed == 'true'
177+
run: |
178+
helm upgrade autogpt-builder ./autogpt-builder \
179+
--namespace ${{ env.NAMESPACE }} \
180+
-f autogpt-builder/values.yaml \
181+
-f autogpt-builder/values.prod.yaml \
182+
--set image.tag=${{ github.sha }}

‎.github/workflows/platform-autogpt-deploy.yaml

+34
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,42 @@ env:
1919
NAMESPACE: dev-agpt
2020

2121
jobs:
22+
migrate:
23+
environment: develop
24+
name: Run migrations for AutoGPT Platform
25+
runs-on: ubuntu-latest
26+
27+
steps:
28+
- name: Checkout code
29+
uses: actions/checkout@v2
30+
31+
- name: Set up Python
32+
uses: actions/setup-python@v4
33+
with:
34+
python-version: '3.11'
35+
36+
- name: Install Python dependencies
37+
run: |
38+
python -m pip install --upgrade pip
39+
pip install prisma
40+
41+
- name: Run Backend Migrations
42+
working-directory: ./autogpt_platform/backend
43+
run: |
44+
python -m prisma migrate deploy
45+
env:
46+
DATABASE_URL: ${{ secrets.BACKEND_DATABASE_URL }}
47+
48+
- name: Run Market Migrations
49+
working-directory: ./autogpt_platform/market
50+
run: |
51+
python -m prisma migrate deploy
52+
env:
53+
DATABASE_URL: ${{ secrets.MARKET_DATABASE_URL }}
54+
2255
build-push-deploy:
2356
name: Build, Push, and Deploy
57+
needs: migrate
2458
runs-on: ubuntu-latest
2559

2660
steps:

‎autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/store.py

+8-2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from redis import Redis
77
from backend.executor.database import DatabaseManager
88

9+
from autogpt_libs.utils.cache import thread_cached_property
910
from autogpt_libs.utils.synchronize import RedisKeyedMutex
1011

1112
from .types import (
@@ -18,9 +19,14 @@
1819

1920

2021
class SupabaseIntegrationCredentialsStore:
21-
def __init__(self, redis: "Redis", db: "DatabaseManager"):
22-
self.db_manager: DatabaseManager = db
22+
def __init__(self, redis: "Redis"):
2323
self.locks = RedisKeyedMutex(redis)
24+
25+
@thread_cached_property
26+
def db_manager(self) -> "DatabaseManager":
27+
from backend.executor.database import DatabaseManager
28+
from backend.util.service import get_service_client
29+
return get_service_client(DatabaseManager)
2430

2531
def add_creds(self, user_id: str, credentials: Credentials) -> None:
2632
with self.locked_user_metadata(user_id):

‎autogpt_platform/backend/backend/util/cache.py ‎autogpt_platform/autogpt_libs/autogpt_libs/utils/cache.py

File renamed without changes.

‎autogpt_platform/backend/.env.example

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@ PYRO_HOST=localhost
2020
SENTRY_DSN=
2121

2222
## User auth with Supabase is required for any of the 3rd party integrations with auth to work.
23-
ENABLE_AUTH=false
23+
ENABLE_AUTH=true
2424
SUPABASE_URL=http://localhost:8000
2525
SUPABASE_SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q
2626
SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long
2727

2828
# For local development, you may need to set FRONTEND_BASE_URL for the OAuth flow for integrations to work.
29-
# FRONTEND_BASE_URL=http://localhost:3000
29+
FRONTEND_BASE_URL=http://localhost:3000
3030

3131
## == INTEGRATION CREDENTIALS == ##
3232
# Each set of server side credentials is required for the corresponding 3rd party

‎autogpt_platform/backend/README.md

+3-2
Original file line numberDiff line numberDiff line change
@@ -58,17 +58,18 @@ We use the Poetry to manage the dependencies. To set up the project, follow thes
5858
6. Migrate the database. Be careful because this deletes current data in the database.
5959

6060
```sh
61-
docker compose up db redis -d
61+
docker compose up db -d
6262
poetry run prisma migrate deploy
6363
```
6464

6565
## Running The Server
6666

6767
### Starting the server without Docker
6868

69-
Run the following command to build the dockerfiles:
69+
Run the following command to run database in docker but the application locally:
7070

7171
```sh
72+
docker compose --profile local up deps --build --detach
7273
poetry run app
7374
```
7475

‎autogpt_platform/backend/backend/blocks/__init__.py

+14-10
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import os
33
import re
44
from pathlib import Path
5+
from typing import Type, TypeVar
56

67
from backend.data.block import Block
78

@@ -24,28 +25,31 @@
2425
AVAILABLE_MODULES.append(module)
2526

2627
# Load all Block instances from the available modules
27-
AVAILABLE_BLOCKS = {}
28+
AVAILABLE_BLOCKS: dict[str, Type[Block]] = {}
2829

2930

30-
def all_subclasses(clz):
31-
subclasses = clz.__subclasses__()
31+
T = TypeVar("T")
32+
33+
34+
def all_subclasses(cls: Type[T]) -> list[Type[T]]:
35+
subclasses = cls.__subclasses__()
3236
for subclass in subclasses:
3337
subclasses += all_subclasses(subclass)
3438
return subclasses
3539

3640

37-
for cls in all_subclasses(Block):
38-
name = cls.__name__
41+
for block_cls in all_subclasses(Block):
42+
name = block_cls.__name__
3943

40-
if cls.__name__.endswith("Base"):
44+
if block_cls.__name__.endswith("Base"):
4145
continue
4246

43-
if not cls.__name__.endswith("Block"):
47+
if not block_cls.__name__.endswith("Block"):
4448
raise ValueError(
45-
f"Block class {cls.__name__} does not end with 'Block', If you are creating an abstract class, please name the class with 'Base' at the end"
49+
f"Block class {block_cls.__name__} does not end with 'Block', If you are creating an abstract class, please name the class with 'Base' at the end"
4650
)
4751

48-
block = cls()
52+
block = block_cls.create()
4953

5054
if not isinstance(block.id, str) or len(block.id) != 36:
5155
raise ValueError(f"Block ID {block.name} error: {block.id} is not a valid UUID")
@@ -87,6 +91,6 @@ def all_subclasses(clz):
8791
if block.disabled:
8892
continue
8993

90-
AVAILABLE_BLOCKS[block.id] = block
94+
AVAILABLE_BLOCKS[block.id] = block_cls
9195

9296
__all__ = ["AVAILABLE_MODULES", "AVAILABLE_BLOCKS"]

0 commit comments

Comments
 (0)