update workflow #2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy and Run PySpark Job on Dataproc | |
| on: | |
| push: | |
| branches: | |
| - main | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| # Authenticate using the correct auth action | |
| - name: Authenticate with Google Cloud | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_KEY }} # Use the secret for your service account key | |
| # Set up Google Cloud SDK without specifying the service account key | |
| - name: Set up Google Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v1 | |
| with: | |
| project_id: "liqour-store-etl" | |
| - name: Install Terraform | |
| run: sudo apt-get install terraform | |
| - name: Initialize Terraform | |
| run: terraform init | |
| - name: Apply Terraform | |
| run: terraform apply -auto-approve | |
| # Step to upload the PySpark job script from GitHub repo to the GCS bucket | |
| - name: Upload PySpark job to GCS | |
| run: | | |
| gsutil cp pyspark-job.py gs://liquor-store-data-bucket/main.py | |
| # Submit the PySpark job to Dataproc | |
| - name: Submit PySpark Job | |
| run: | | |
| gcloud dataproc jobs submit pyspark gs://liquor-store-data-bucket/main.py \ | |
| --cluster=liquor-store-dataproc-cluster --region=europe-west1 |