docker-compose.yml

version: '3'

services:
  source_postgres:
    image: postgres:15-alpine
    ports:
      - '5433:5432'
    networks:
      - pulsaproj-net
    environment:
      POSTGRES_DB: source_db
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: secret
    volumes:
      - ./db/init.sql:/docker-entrypoint-initdb.d/init.sql
      - ./data/01_raw/MyXL-v2.csv:/docker-entrypoint-initdb.d/data/MyXL-v2.csv
      - ./data/01_raw/MyIM3-v2.csv:/docker-entrypoint-initdb.d/data/MyIM3-v2.csv
      - ./data/01_raw/MySF-v2.csv:/docker-entrypoint-initdb.d/data/MySF-v2.csv
      - ./data/01_raw/MyTelkomsel-v2.csv:/docker-entrypoint-initdb.d/data/MyTelkomsel-v2.csv

  destination_postgres:
    image: postgres:15-alpine
    ports: 
      - '5434:5432'
    networks:
      - pulsaproj-net
    environment:
      POSTGRES_DB: destination_db
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: secret
  
  # store airflow metadata
  postgres:
    image: postgres:15-alpine
    networks:
      - pulsaproj-net
    environment:
      - POSTGRES_USER=admin
      - POSTGRES_PASSWORD=secret
      - POSTGRES_DB=airflow

  extraction:
    build:
      context: ./extract # Directory containing the Dockerfile and to_lake.py
      dockerfile: Dockerfile # Name of the Dockerfile, if it's something other than "Dockerfile", specify here
    image: to_lake:latest
    command: [ "python", "to_lake.py" ]
    networks:
      - pulsaproj-net
    depends_on:
      - source_postgres

  kedro_pipeline:
    build:
      context: .
      dockerfile: Dockerfile 
    image: kedro-pipeline:latest
    networks:
      - pulsaproj-net
    volumes:
      # - .:/app  # Mount the current directory to /app in the container for development
      - ./conf/:/app/conf 
      - ./data/:/app/data
      - ./src/:/app/src
      - ./pyproject.toml:/app/pyproject.toml
    depends_on:
      - source_postgres
      - destination_postgres
      - extraction

  init-airflow:
    image: apache/airflow:latest
    depends_on:
      - postgres
    networks:
      - pulsaproj-net
    environment:
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://admin:secret@postgres/airflow
    command: >
      bash -c "airflow db init && 
              airflow users create --username anggapark --password secret --firstname angga --lastname wibowo --role Admin --email anggapark@gmail.com"

  webserver:
    build:
      context: .
      dockerfile: airflow/Dockerfile
    user: root
    depends_on:
      - postgres
      - source_postgres
      - destination_postgres
    networks:
      - pulsaproj-net
    extra_hosts:
      - "host.docker.internal:host-gateway"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./extract:/opt/airflow/extract # extract dir
      - ./conf/:/opt/conf 
      - ./data/:/opt/data
      - ./src/:/opt/src
      - ./pyproject.toml:/opt/pyproject.toml
      - /var/run/docker.sock:/var/run/docker.sock
    environment:
      - LOAD_EX=n
      - EXECUTOR=Local
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://admin:secret@postgres/airflow
      - AIRFLOW_CONN_SOURCE_POSTGRES=postgresql://postgres:secret@source_postgres:5433/source_db
      - AIRFLOW_CONN_DESTINATION_POSTGRES=postgres://postgres:secret@destination_postgres:5434/destination_db
      - AIRFLOW__CORE__FERNET_KEY=plIipb9RU3-3wJ1UNaAtqVNJrqFEks1-dGbJM34EW7U=
      - AIRFLOW__WEBSERVER__DEFAULT_USER_USERNAME=anggapark
      - AIRFLOW__WEBSERVER__DEFAULT_USER_PASSWORD=secret
      - AIRFLOW_WWW_USER_USERNAME=anggapark
      - AIRFLOW_WWW_USER_PASSWORD=secret
      - AIRFLOW__WEBSERVER__SECRET_KEY=secret
    ports:
      - "8080:8080"
    command: webserver

  scheduler:
    build:
      context: .
      dockerfile: airflow/Dockerfile
    user: root
    depends_on:
      - postgres
    networks:
      - pulsaproj-net
    extra_hosts:
      - "host.docker.internal:host-gateway"
    volumes:
      # - .:/opt # kedro dir
      - ./airflow/dags:/opt/airflow/dags
      - ./extract:/opt/airflow/extract # extract dir
      - ./conf/:/opt/conf 
      - ./data/:/opt/data
      - ./src/:/opt/src
      - ./pyproject.toml:/opt/pyproject.toml
      - /var/run/docker.sock:/var/run/docker.sock
    environment:
      - LOAD_EX=n
      - EXECUTOR=Local
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://admin:secret@postgres/airflow
      - AIRFLOW_CONN_DESTINATION_POSTGRES=postgres://postgres:secret@destination_postgres:5434/destination_db
      - AIRFLOW__CORE__FERNET_KEY=plIipb9RU3-3wJ1UNaAtqVNJrqFEks1-dGbJM34EW7U=
      - AIRFLOW__WEBSERVER__SECRET_KEY=secret
      - AIRFLOW_WWW_USER_USERNAME=anggapark
      - AIRFLOW_WWW_USER_PASSWORD=secret
    command: scheduler

networks:
  pulsaproj-net:
    name: pulsaproj-net
    driver: bridge

volumes:
  destination_db_data: