compose.yaml

services:
  api:
    build: ./api
    environment:
      - DB_USER=${POSTGRES_USER}
      - DB_PASSWORD=${POSTGRES_PASSWORD}
      - DB_NAME=${POSTGRES_DB}
    ports:
      - 8010:80
    depends_on:
      web_scraper:
        condition: service_started
      db:
        condition: service_healthy

  web_scraper:
    build: ./web_scraping
    environment:
      - DB_USER=${POSTGRES_USER}
      - DB_PASSWORD=${POSTGRES_PASSWORD}
      - DB_NAME=${POSTGRES_DB}
    volumes:
      - web-scraper-flag:/app/data
    depends_on:
      db:
        condition: service_healthy

  db:
    image: postgres
    restart: unless-stopped
    volumes:
      - db-data:/var/lib/postgresql/data
    environment:
      - POSTGRES_USER=${POSTGRES_USER}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - POSTGRES_DB=${POSTGRES_DB}
    expose:
      - 5432
    healthcheck:
      test: [ "CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}" ]
      interval: 10s
      timeout: 5s
      retries: 5

  minio:
    image: minio/minio
    volumes:
      - minio-data:/data
    environment:
      - MINIO_ROOT_USER=${MINIO_ROOT_USER}
      - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}
    command: server --console-address ":9090" /data
    ports:
      - 9000:9000
      - 9090:9090

  minio-setup:
    image: minio/mc
    depends_on:
      - minio
    entrypoint: >
      /bin/sh -c "
      until mc config host add myminio http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD}; do
        echo 'Waiting for MinIO...'
        sleep 5
      done;
      mc mb myminio/${MINIO_BUCKET_NAME};
      mc admin user add myminio ${MINIO_NEW_USER} ${MINIO_NEW_USER_PASSWORD};
      mc admin policy attach myminio readwrite --user=${MINIO_NEW_USER};
      "

  custom-airbyte-connector:
    environment:
      - AIRBYTE_HOST=http://${HOST}
    build:
      context: ./extract-load-transform/custom-airbyte-connector
      dockerfile: Dockerfile
    image: airbyte/source-roman-coins-api:latest
    pull_policy: build

  airbyte-configurator:
    depends_on:
      - api
      - minio-setup
    environment:
      - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME}
      - MINIO_NEW_USER=${MINIO_NEW_USER}
      - MINIO_NEW_USER_PASSWORD=${MINIO_NEW_USER_PASSWORD}
      - AIRBYTE_HOST=http://${HOST}
      - AIRBYTE_USERNAME=${AIRBYTE_USERNAME}
      - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD}
      - CONNECTOR_START_DATE=${CONNECTOR_START_DATE}
    build: 
      context: ./extract-load-transform/airbyte-api-minio-connection
      dockerfile: Dockerfile
    pull_policy: build

  dagster-webserver:
    depends_on:
      db:
        condition: service_healthy
      airbyte-configurator:
        condition: service_completed_successfully
    volumes:
      - duckdb-data:/opt/dagster-dbt/app/data
    environment:
      - POSTGRES_USER=${POSTGRES_USER}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - POSTGRES_DB=${POSTGRES_DB}
      - DB_HOST=db
      - AIRBYTE_USERNAME=${AIRBYTE_USERNAME}
      - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD}
      - HOST=${HOST}
      - MINIO_ENDPOINT=${HOST}:9000
      - DUCKDB_DATABASE=${DUCKDB_DATABASE}
      - LOADING_TABLE=${LOADING_TABLE}
      - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME}
      - MINIO_USER=${MINIO_NEW_USER}
      - MINIO_PASSWORD=${MINIO_NEW_USER_PASSWORD}
    build: 
      context: ./extract-load-transform/dagster-dbt
      dockerfile: Dockerfile
    entrypoint:
      - dagster-webserver
      - -h
      - "0.0.0.0"
      - -p
      - "3000"
      - -w
      - workspace.yaml
    expose:
      - 3000
    ports:
      - 3000:3000

  dagster-daemon:
    depends_on:
      db:
        condition: service_healthy
      airbyte-configurator:
        condition: service_completed_successfully
    volumes:
      - duckdb-data:/opt/dagster-dbt/app/data
    environment:
      - POSTGRES_USER=${POSTGRES_USER}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - POSTGRES_DB=${POSTGRES_DB}
      - DB_HOST=db
      - AIRBYTE_USERNAME=${AIRBYTE_USERNAME}
      - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD}
      - HOST=${HOST}
      - MINIO_ENDPOINT=${HOST}:9000
      - DUCKDB_DATABASE=${DUCKDB_DATABASE}
      - LOADING_TABLE=${LOADING_TABLE}
      - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME}
      - MINIO_USER=${MINIO_NEW_USER}
      - MINIO_PASSWORD=${MINIO_NEW_USER_PASSWORD}
    build: 
      context: ./extract-load-transform/dagster-dbt
      dockerfile: Dockerfile
    entrypoint:
      - dagster-daemon
      - run
    restart: on-failure

volumes:
  db-data:
  web-scraper-flag:
  minio-data:
  duckdb-data: