Full refresh

snowplow · Nov 8, 2024 · 8dfa9f0 · 8dfa9f0
1 parent 08010cc
commit 8dfa9f0
Show file tree

Hide file tree

Showing 4 changed files with 132 additions and 20 deletions.
diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml
@@ -16,14 +16,117 @@ jobs:
   list_s3_objects:
     name: list_s3_objects
     runs-on: ubuntu-latest
-
+    defaults:
+      run:
+        # Run tests from integration_tests sub dir
+        working-directory: ./integration_tests
+    strategy:
+      fail-fast: false
+      matrix:
+        dbt_version: ["1.*"]
+        warehouse: ["spark_iceberg"]
     steps:
     - name: Configure AWS CLI with OIDC role assumption
       uses: aws-actions/configure-aws-credentials@v4
       with:
         role-to-assume: ${{ env.AWS_ROLE_ARN }}
         aws-region: ${{ env.AWS_REGION }}
 
+    - name: Extract AWS credentials
+      run: |
+        echo "AWS_ACCESS_KEY_ID=$(aws configure get aws_access_key_id)" >> $GITHUB_ENV
+        echo "AWS_SECRET_ACCESS_KEY=$(aws configure get aws_secret_access_key)" >> $GITHUB_ENV
+        echo "AWS_SESSION_TOKEN=$(aws configure get aws_session_token)" >> $GITHUB_ENV
+
     - name: List objects from S3 bucket
       run: |
-        aws s3 ls s3://${{ env.S3_BUCKET }} --recursive
+        aws s3 ls s3://${{ env.S3_BUCKET }} --recursive
+
+    # You can now use the credentials in subsequent steps like this:
+    - name: Example Next Step
+      env:
+        AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+        AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
+      run: |
+        # Your commands here will have access to the temporary credentials
+        echo "Credentials are available in this step"
+
+    - name: Check out
+      uses: actions/checkout@v4
+
+    # Remove '*' and replace '.' with '_' in DBT_VERSION & set as SCHEMA_SUFFIX.
+    # SCHEMA_SUFFIX allows us to run multiple versions of dbt in parallel without overwriting the output tables
+    - name: Set SCHEMA_SUFFIX env
+      run: echo "SCHEMA_SUFFIX=$(echo ${DBT_VERSION%.*} | tr . _)" >> $GITHUB_ENV
+      env:
+        DBT_VERSION: ${{ matrix.dbt_version }}
+    - name: Configure Docker credentials
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKERHUB_SNOWPLOWCI_READ_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_SNOWPLOWCI_READ_PASSWORD }}
+    - name: Set warehouse variables
+      id: set_warehouse
+      run: |
+        WAREHOUSE_PLATFORM=$(echo ${{ matrix.warehouse }} | cut -d'_' -f1)
+        WAREHOUSE_SPECIFIC=$(echo ${{ matrix.warehouse }} | cut -s -d'_' -f2)
+        echo "WAREHOUSE_PLATFORM=${WAREHOUSE_PLATFORM}" >> $GITHUB_ENV
+        echo "WAREHOUSE_SPECIFIC=${WAREHOUSE_SPECIFIC}" >> $GITHUB_ENV
+        echo "warehouse_platform=${WAREHOUSE_PLATFORM}" >> $GITHUB_OUTPUT
+        echo "warehouse_specific=${WAREHOUSE_SPECIFIC}" >> $GITHUB_OUTPUT
+    # Remove '*' and replace '.' with '_' in DBT_VERSION & set as SCHEMA_SUFFIX.
+    # SCHEMA_SUFFIX allows us to run multiple versions of dbt in parallel without overwriting the output tables
+    - name: Set SCHEMA_SUFFIX env
+      run: echo "SCHEMA_SUFFIX=$(echo ${DBT_VERSION%.*} | tr . _)" >> $GITHUB_ENV
+      env:
+        DBT_VERSION: ${{ matrix.dbt_version }}
+
+    - name: Set DEFAULT_TARGET env
+      run: |
+        echo "DEFAULT_TARGET=${{matrix.warehouse}}" >> $GITHUB_ENV
+
+    - name: Python setup
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.8.x"
+
+    - name: Pip cache
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ matrix.dbt_version }}-${{env.WAREHOUSE_PLATFORM}}
+        restore-keys: |
+          ${{ runner.os }}-pip-${{ matrix.dbt_version }}-${{env.WAREHOUSE_PLATFORM}}
+
+    - name: Install spark dependencies
+      run: |
+        pip install --upgrade pip wheel setuptools
+        pip install -Iv "dbt-${{ env.WAREHOUSE_PLATFORM }}[PyHive]"==${{ matrix.dbt_version }} --upgrade
+        dbt deps
+      if: ${{env.WAREHOUSE_PLATFORM == 'spark'}}
+
+    - name: Install Docker Compose
+      run: |
+        sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+        sudo chmod +x /usr/local/bin/docker-compose
+
+    
+    - name: Build and start Spark cluster
+      working-directory: .github/workflows/spark_deployment
+      run: |
+        docker-compose up -d
+        echo "Waiting for Spark services to start..."
+        sleep 90
+      if: ${{env.WAREHOUSE_PLATFORM == 'spark'}}
+
+    - name: "Pre-test: Drop ci schemas"
+      run: |
+        dbt run-operation post_ci_cleanup --target ${{matrix.warehouse}}
+
+    - name: Run tests
+      run: ./.scripts/integration_test.sh -d ${{matrix.warehouse}}
+
+    - name: "Post-test: Drop ci schemas"
+      run: |
+        dbt run-operation post_ci_cleanup --target ${{matrix.warehouse}}
diff --git a/.github/workflows/spark_deployment/docker-compose.yml b/.github/workflows/spark_deployment/docker-compose.yml
@@ -7,7 +7,6 @@ networks:
 services:
   spark-master:
     image: snowplow/spark-s3-iceberg:latest
-    command: ["/bin/bash", "-c", "/spark/sbin/start-master.sh -h spark-master --properties-file /spark/conf/spark-defaults.conf && tail -f /spark/logs/spark--org.apache.spark.deploy.master.Master-1-*.out"]
     hostname: spark-master
     ports:
       - '8080:8080'
@@ -16,51 +15,52 @@ services:
       - SPARK_LOCAL_IP=spark-master
       - SPARK_MASTER_HOST=spark-master
       - SPARK_MASTER_PORT=7077
-      - SPARK_MASTER_OPTS="-Dspark.driver.memory=2g"
+      # AWS credentials
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
-      - AWS_REGION=eu-west-1
-      - AWS_DEFAULT_REGION=eu-west-1
     volumes:
-      - ./spark-defaults.conf:/spark/conf/spark-defaults.conf
+      - ./spark-defaults.conf:/spark/conf/spark-defaults.conf.template
+      - ./setup.sh:/setup.sh
+    entrypoint: ["/bin/bash", "/setup.sh"]
+    command: ["/bin/bash", "-c", "/spark/sbin/start-master.sh -h spark-master --properties-file /spark/conf/spark-defaults.conf && tail -f /spark/logs/spark--org.apache.spark.deploy.master.Master-1-*.out"]
     networks:
       - spark-network
 
   spark-worker:
     image: snowplow/spark-s3-iceberg:latest
-    command: ["/bin/bash", "-c", "sleep 10 && /spark/sbin/start-worker.sh spark://spark-master:7077 --properties-file /spark/conf/spark-defaults.conf && tail -f /spark/logs/spark--org.apache.spark.deploy.worker.Worker-*.out"]
     depends_on:
       - spark-master
     environment:
-      - SPARK_WORKER_CORES=2
-      - SPARK_WORKER_MEMORY=4G
-      - SPARK_EXECUTOR_MEMORY=3G
       - SPARK_LOCAL_IP=spark-worker
       - SPARK_MASTER=spark://spark-master:7077
+      # AWS credentials
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
-      - AWS_REGION=eu-west-1
-      - AWS_DEFAULT_REGION=eu-west-1
     volumes:
-      - ./spark-defaults.conf:/spark/conf/spark-defaults.conf
+      - ./spark-defaults.conf:/spark/conf/spark-defaults.conf.template
+      - ./setup.sh:/setup.sh
+    entrypoint: ["/bin/bash", "/setup.sh"]
+    command: ["/bin/bash", "-c", "sleep 10 && /spark/sbin/start-worker.sh spark://spark-master:7077 --properties-file /spark/conf/spark-defaults.conf && tail -f /spark/logs/spark--org.apache.spark.deploy.worker.Worker-*.out"]
     networks:
       - spark-network
 
   thrift-server:
     image: snowplow/spark-s3-iceberg:latest
-    command: ["/bin/bash", "-c", "sleep 30 && /spark/sbin/start-thriftserver.sh --master spark://spark-master:7077 --driver-memory 2g --executor-memory 3g --hiveconf hive.server2.thrift.port=10000 --hiveconf hive.server2.thrift.bind.host=0.0.0.0 --conf spark.sql.hive.thriftServer.async=true --conf spark.sql.hive.thriftServer.workerQueue.size=2000 --conf spark.sql.hive.thriftServer.maxWorkerThreads=100 --conf spark.sql.hive.thriftServer.minWorkerThreads=50 && tail -f /spark/logs/spark--org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-*.out"]
     ports:
       - '10000:10000'
+      - '4040:4040'
     depends_on:
       - spark-master
       - spark-worker
     environment:
       - SPARK_LOCAL_IP=thrift-server
+      # AWS credentials
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
-      - AWS_REGION=eu-west-1
-      - AWS_DEFAULT_REGION=eu-west-1
     volumes:
-      - ./spark-defaults.conf:/spark/conf/spark-defaults.conf
+      - ./spark-defaults.conf:/spark/conf/spark-defaults.conf.template
+      - ./setup.sh:/setup.sh
+    entrypoint: ["/bin/bash", "/setup.sh"]
+    command: ["/bin/bash", "-c", "sleep 30 && /spark/sbin/start-thriftserver.sh --master spark://spark-master:7077 --driver-memory 2g --executor-memory 3g --hiveconf hive.server2.thrift.port=10000 --hiveconf hive.server2.thrift.bind.host=0.0.0.0 --conf spark.sql.hive.thriftServer.async=true --conf spark.sql.hive.thriftServer.workerQueue.size=2000 --conf spark.sql.hive.thriftServer.maxWorkerThreads=100 --conf spark.sql.hive.thriftServer.minWorkerThreads=50 && tail -f /spark/logs/spark--org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-*.out"]
     networks:
       - spark-network
diff --git a/.github/workflows/spark_deployment/setup.sh b/.github/workflows/spark_deployment/setup.sh
@@ -0,0 +1,9 @@
+
+#!/bin/bash
+
+# Create a new spark-defaults.conf with substituted values
+sed -e "s|\${AWS_ACCESS_KEY_ID}|$AWS_ACCESS_KEY_ID|g" \
+    -e "s|\${AWS_SECRET_ACCESS_KEY}|$AWS_SECRET_ACCESS_KEY|g" \
+    /spark/conf/spark-defaults.conf.template > /spark/conf/spark-defaults.conf
+# Execute the passed command
+exec "$@"
diff --git a/.github/workflows/spark_deployment/spark-defaults.conf b/.github/workflows/spark_deployment/spark-defaults.conf
@@ -9,8 +9,8 @@ spark.sql.defaultCatalog                       glue
 spark.sql.catalog.glue.database                dbt-spark-iceberg
 
 spark.hadoop.fs.s3a.impl                       org.apache.hadoop.fs.s3a.S3AFileSystem
-spark.hadoop.fs.s3a.access.key                 <AWS_ACCESS_KEY_ID>
-spark.hadoop.fs.s3a.secret.key                 <AWS_SECRET_ACCESS_KEY>
+spark.hadoop.fs.s3a.access.key                 ${AWS_ACCESS_KEY_ID}
+spark.hadoop.fs.s3a.secret.key                 ${AWS_SECRET_ACCESS_KEY}
 spark.hadoop.fs.s3a.endpoint                   s3.eu-west-1.amazonaws.com
 spark.hadoop.fs.s3a.path.style.access          true
 spark.hadoop.fs.s3a.region                     eu-west-1