From 15a89327858dc83a19f21c56b09dd1e12389f76e Mon Sep 17 00:00:00 2001 From: Mohit Goyal Date: Fri, 20 Dec 2024 16:13:19 +0530 Subject: [PATCH 1/5] adds infra docker compose yaml --- docker/infra.docker-compose.yml | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docker/infra.docker-compose.yml diff --git a/docker/infra.docker-compose.yml b/docker/infra.docker-compose.yml new file mode 100644 index 000000000..ba28bbbbf --- /dev/null +++ b/docker/infra.docker-compose.yml @@ -0,0 +1,41 @@ +version: "3.9" + +services: + db: + image: postgres:11 + environment: + - POSTGRES_DB=${POSTGRES_DB:-db} + - POSTGRES_USER=${POSTGRES_USER:-user} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-pass} + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + networks: + - mynetwork + + redis: + image: "redis:alpine" + ports: + - "6379:6379" + networks: + - mynetwork + + fluentd: + build: + context: ./fluentd + dockerfile: Dockerfile + image: fluent/fluentd:v1.12-debian-1 + volumes: + - ./fluentd/fluentd.conf:/fluentd/etc/fluent.conf + ports: + - "24224:24224" + - "24224:24224/udp" + networks: + - mynetwork + +networks: + mynetwork: + +volumes: + postgres_data: From d3f7b66b77756e1cbbf92e405779a7798854fb97 Mon Sep 17 00:00:00 2001 From: Mohit Goyal Date: Fri, 20 Dec 2024 16:20:43 +0530 Subject: [PATCH 2/5] adds all server start script --- scripts/start_all_services.sh | 79 +++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 scripts/start_all_services.sh diff --git a/scripts/start_all_services.sh b/scripts/start_all_services.sh new file mode 100644 index 000000000..78367392b --- /dev/null +++ b/scripts/start_all_services.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# Exit script on any error +set -e + +# Step 1: Set environment variables (adjust as needed) +export DJANGO_DEBUG=True +export CELERY_BROKER_URL=redis://localhost:6379/0 +export CELERY_RESULT_BACKEND=redis://localhost:6379/0 +export REDIS_URL=redis://localhost:6379/0 +export POSTGRES_DB=db +export POSTGRES_USER=user +export POSTGRES_PASSWORD=pass +export POSTGRES_HOST=localhost +export POSTGRES_PORT=5432 +export FLUENTD_ADDRESS=localhost:24224 +# export OKTA_DOMAIN=your-okta-domain +# export OKTA_CLIENT_ID=your-okta-client-id + +# Step 2: Set up Python virtual environment + +echo "Setting up Python virtual environment..." + +# Check if virtualenv is installed, if not, install it +if ! command -v virtualenv &>/dev/null; then + echo "virtualenv not found, installing..." + pip install virtualenv +fi + +# Create and activate the virtual environment +python3 -m venv venv +source venv/bin/activate + +# Step 3: Install Python dependencies +echo "Installing Python dependencies..." +pip install -r requirements.txt + +# Step 4: Install frontend dependencies +echo "Installing frontend dependencies..." +cd web +npm install + +# Step 5: Start the services + +# Start Django server +echo "Starting Django server..." +cd .. +python manage.py runserver 0.0.0.0:8080 & + +# Start Celery workers +echo "Starting Celery workers..." +celery -A playbooks worker --concurrency=10 -l INFO -Ofair -Q celery --max-tasks-per-child=1000 --prefetch-multiplier=1 & + +# Start Workflow Scheduler Celery Worker +echo "Starting Workflow Scheduler Celery Worker..." +celery -A playbooks worker --concurrency=10 -l INFO -Ofair -Q workflow_scheduler --max-tasks-per-child=1000 --prefetch-multiplier=1 & + +# Start Workflow Executor Celery Worker +echo "Starting Workflow Executor Celery Worker..." +celery -A playbooks worker --concurrency=10 -l INFO -Ofair -Q workflow_executor --max-tasks-per-child=1000 --prefetch-multiplier=1 & + +# Start Workflow Action Execution Celery Worker +echo "Starting Workflow Action Execution Celery Worker..." +celery -A playbooks worker --concurrency=10 -l INFO -Ofair -Q workflow_action_execution --max-tasks-per-child=1000 --prefetch-multiplier=1 & + +# Start Celery Beat +echo "Starting Celery Beat..." +celery -A playbooks beat --loglevel=info & + +# Start the Web frontend +echo "Starting the Web frontend..." +cd web +npm run build && npm run start & + +# Step 6: Verify services are running +echo "All services started successfully. Monitoring logs..." + +# Keep the script running so that the services remain active +tail -f /dev/null From b33927b578d056ff4023b98153b0de986fd472d2 Mon Sep 17 00:00:00 2001 From: Mohit Goyal Date: Fri, 20 Dec 2024 16:29:50 +0530 Subject: [PATCH 3/5] moves files --- docker/infra.docker-compose.yml => infra.docker-compose.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker/infra.docker-compose.yml => infra.docker-compose.yml (100%) diff --git a/docker/infra.docker-compose.yml b/infra.docker-compose.yml similarity index 100% rename from docker/infra.docker-compose.yml rename to infra.docker-compose.yml From 278427201bd37098223705879d424214df15f120 Mon Sep 17 00:00:00 2001 From: Mohit Goyal Date: Fri, 20 Dec 2024 16:35:16 +0530 Subject: [PATCH 4/5] moves script --- scripts/start_all_services.sh => start_all_services.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/start_all_services.sh => start_all_services.sh (100%) mode change 100644 => 100755 diff --git a/scripts/start_all_services.sh b/start_all_services.sh old mode 100644 new mode 100755 similarity index 100% rename from scripts/start_all_services.sh rename to start_all_services.sh From 94f76585a7b38127ce68062906c321833e1848f1 Mon Sep 17 00:00:00 2001 From: Mohit Goyal Date: Fri, 20 Dec 2024 16:46:01 +0530 Subject: [PATCH 5/5] updgrades script --- start_all_services.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/start_all_services.sh b/start_all_services.sh index 78367392b..653ea9f27 100755 --- a/start_all_services.sh +++ b/start_all_services.sh @@ -21,18 +21,18 @@ export FLUENTD_ADDRESS=localhost:24224 echo "Setting up Python virtual environment..." -# Check if virtualenv is installed, if not, install it -if ! command -v virtualenv &>/dev/null; then - echo "virtualenv not found, installing..." - pip install virtualenv +# Create the virtual environment using python3 -m venv +if [ ! -d "venv" ]; then + python3 -m venv venv + echo "Virtual environment created successfully." fi -# Create and activate the virtual environment -python3 -m venv venv +# Activate the virtual environment source venv/bin/activate # Step 3: Install Python dependencies echo "Installing Python dependencies..." +pip install --upgrade pip # Upgrade pip to avoid any issues with older versions pip install -r requirements.txt # Step 4: Install frontend dependencies