update readme #21
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy and Run PySpark Job on Dataproc | |
| on: | |
| push: | |
| branches: | |
| - main | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| # Authenticate using the correct auth action | |
| - name: Authenticate with Google Cloud | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_KEY }} | |
| # Set up Google Cloud SDK | |
| - name: Set up Google Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v1 | |
| with: | |
| project_id: "liqour-store-etl" | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v1 | |
| with: | |
| terraform_version: 1.5.7 | |
| - name: Terraform init and validate | |
| run: | | |
| terraform init | |
| terraform validate | |
| - name: Terraform Plan | |
| id: plan | |
| run: terraform plan -out=plan.out | |
| continue-on-error: true # Ensure the job continues even if no changes | |
| # Conditional Terraform apply step based on the plan output | |
| - name: Check if changes need to be applied | |
| if: ${{ steps.plan.outcome == 'success' && steps.plan.conclusion == 'success' }} | |
| run: | | |
| terraform apply -auto-approve plan.out | |
| # Skip apply if no changes (plan success with no diffs) | |
| - name: Skip apply if no changes | |
| if: ${{ steps.plan.outcome != 'success' }} | |
| run: echo "No changes to apply." | |
| # Step to upload the PySpark job script from GitHub repo to the GCS bucket | |
| - name: Upload PySpark job to GCS | |
| if: ${{ steps.plan.conclusion == 'success' }} | |
| run: | | |
| gsutil cp main.py gs://liquor-store-bucket/main.py | |
| # Instantiate the Dataproc workflow template | |
| - name: Run Dataproc Workflow Template | |
| if: ${{ steps.plan.conclusion == 'success' }} | |
| run: | | |
| gcloud dataproc workflow-templates instantiate liquor-store-etl-workflow \ | |
| --region us-central1 |