deploy pollution v2 into production

.github/workflows/ci-pollution-v2.yml

@@ -0,0 +1,232 @@
+name: CI/CD pollution-v2
+
+on:
+  push:
+    paths:
+      - "pollution_v2/**"
+      - ".github/workflows/ci-pollution-v2.yml"
+      - "!*.md"
+
+env:
+  WORKING_DIRECTORY: pollution_v2
+  PROJECT_NAME: odh-mobility-el-pollution-v2
+  DOCKER_IMAGE: ghcr.io/${{ github.repository }}/odh-mobility-el-pollution-v2
+
+jobs:
+  # Deploy Test
+  deploy-test-pollution-v2:
+    runs-on: ubuntu-24.04
+    if: github.ref == 'refs/heads/main'
+    concurrency: deploy-test-pollution-v2
+    env:
+      KEYCLOAK_URL: https://auth.opendatahub.testingmachine.eu
+      DOCKER_TAG: ${{ github.sha }}-test
+    steps:
+      - name: Checkout source code
+        uses: noi-techpark/github-actions/checkout@v2
+      - name: Create .env file
+        uses: noi-techpark/github-actions/env-file@v2
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}
+        env:
+          # General deployment options
+          X_COMPOSE_PROJECT_NAME: ${{ env.PROJECT_NAME }}
+          X_DOCKER_IMAGE: ${{ env.DOCKER_IMAGE }}
+          X_DOCKER_TAG: ${{ env.DOCKER_TAG }}
+
+          X_LOG_LEVEL: INFO
+          X_LOG_LEVEL_LIBS: INFO
+
+          X_SERVER_PORT: 1097
+
+          X_pollution-v2_TASK_SCHEDULING_MINUTE: "*/10"
+          X_pollution-v2_TASK_SCHEDULING_HOUR: "*"
+
+          X_ODH_BASE_READER_URL: https://mobility.api.opendatahub.testingmachine.eu
+          X_ODH_BASE_WRITER_URL: https://share.opendatahub.testingmachine.eu
+          X_ODH_AUTHENTICATION_URL: ${{ env.KEYCLOAK_URL }}/auth/
+          X_ODH_CLIENT_ID: odh-a22-dataprocessor
+          X_ODH_CLIENT_SECRET: ${{ secrets.COMBINED_CLIENT_SECRET_TEST }}
+          X_ODH_GRANT_TYPE: client_credentials
+          X_ODH_PAGINATION_SIZE: 10000
+          X_ODH_MAX_POST_BATCH_SIZE: 10000
+          X_PROVENANCE_NAME: ${{ env.PROJECT_NAME }}
+          X_PROVENANCE_VERSION: ${{ github.sha }}
+
+          X_AIRFLOW_PROJ_DIR: /opt/pollution-v2
+          X_AIRFLOW_UID: 5000
+          X_AIRFLOW__CORE__EXECUTOR: CeleryExecutor
+          X_AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
+          X_AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow
+          X_AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
+          X_AIRFLOW__CORE__FERNET_KEY: ''
+          X_AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
+          X_AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
+          X_AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session'
+          # yamllint disable rule:line-length
+          # Use simple http server on scheduler for health checks
+          # See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server
+          # yamllint enable rule:line-length
+          X_AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'
+          # WARNING: Use _PIP_ADDITIONAL_REQUIREMENTS option ONLY for a quick checks
+          # for other purpose (development, test and especially production usage) build/extend Airflow image.
+          X__PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
+          X_AIRFLOW_VAR_ODH_AUTHENTICATION_URL: ${{ env.KEYCLOAK_URL }}/auth/
+          X_AIRFLOW_VAR_ODH_BASE_READER_URL: 'https://mobility.api.opendatahub.testingmachine.eu'
+          X_AIRFLOW_VAR_ODH_BASE_WRITER_URL: 'https://share.opendatahub.testingmachine.eu'
+          X_AIRFLOW_VAR_ODH_CLIENT_ID: 'odh-a22-dataprocessor'
+          X_AIRFLOW_VAR_ODH_CLIENT_SECRET: ${{ secrets.COMBINED_CLIENT_SECRET_TEST }}
+          X_AIRFLOW_VAR_ODH_GRANT_TYPE: client_credentials
+          X_AIRFLOW_VAR_ODH_MAX_POST_BATCH_SIZE: '10000'
+          X_AIRFLOW_VAR_ODH_PAGINATION_SIZE: '10000'
+          X_AIRFLOW_WWW_USER_USERNAME: ${{ secrets.AIRFLOW_WWW_USER_USERNAME }}
+          X_AIRFLOW_WWW_USER_PASSWORD: ${{ secrets.AIRFLOW_WWW_USER_PASSWORD }}
+          X_AIRFLOW_VAR_DATATYPE_PREFIX: ''
+
+          X_AIRFLOW_VAR_DAG_POLLUTION_EXECUTION_CRONTAB: 0 4 * * *
+          X_AIRFLOW_VAR_DAG_VALIDATION_EXECUTION_CRONTAB: 0 2 * * *
+
+          # username/pass not needed with client_credentials
+          X_AIRFLOW_VAR_ODH_PASSWORD: 'ODH_PASSWORD'
+          X_AIRFLOW_VAR_ODH_USERNAME: 'ODH_USERNAME'
+          X_AIRFLOW_VAR_ODH_MINIMUM_STARTING_DATE: "2017-01-01"
+          X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_HOST: 'redis'
+          X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_PORT: 6379
+          X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_DB: 3
+          # X_AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG: 4
+          X_AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG: 16
+
+          # Currently no logging enabled
+          # X_AIRFLOW__CORE__REMOTE_LOGGING: '<your_choice_on_remote_logging>'
+          # X_AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER: '<your_bucket_for_remote_logging>'
+          # X_AIRFLOW__CORE__REMOTE_LOG_CONN_ID: '<your_connection_id_for_remote_logging>'
+          # X_AIRFLOW__CORE__ENCRYPT_S3_LOGS: '<your_choice_on_encrypt_logs>'
+          # X_AIRFLOW_CONN_MINIO_S3_CONN: '<your_connection_details_for_remote_logging>'
+          X_NO_PROXY: '*'
+
+
+      - name: Build and push images
+        uses: noi-techpark/github-actions/docker-build-and-push@v2
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure
+          docker-username: ${{ github.actor }}
+          docker-password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Deploy application
+        uses: noi-techpark/github-actions/docker-deploy@v2
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure/ansible
+          hosts: 'test'
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+          docker-username: 'noi-techpark-bot'
+          docker-password: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          project-name: ${{ env.PROJECT_NAME }}
+
+  # Deploy Prod
+  deploy-prod-pollution-v2:
+    runs-on: ubuntu-24.04
+    if: github.ref == 'refs/heads/prod'
+    concurrency: deploy-prod-pollution-v2
+    env:
+      KEYCLOAK_URL: https://auth.opendatahub.com
+      DOCKER_TAG: ${{ github.sha }}-prod
+    steps:
+      - name: Checkout source code
+        uses: noi-techpark/github-actions/checkout@v2
+      - name: Create .env file
+        uses: noi-techpark/github-actions/env-file@v2
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}
+        env:
+          # General deployment options
+          X_COMPOSE_PROJECT_NAME: ${{ env.PROJECT_NAME }}
+          X_DOCKER_IMAGE: ${{ env.DOCKER_IMAGE }}
+          X_DOCKER_TAG: ${{ env.DOCKER_TAG }}
+
+          X_LOG_LEVEL: INFO
+          X_LOG_LEVEL_LIBS: INFO
+
+          X_SERVER_PORT: 1098
+
+          X_pollution-v2_TASK_SCHEDULING_MINUTE: "*/10"
+          X_pollution-v2_TASK_SCHEDULING_HOUR: "*"
+
+          X_ODH_BASE_READER_URL: https://mobility.api.opendatahub.com
+          X_ODH_BASE_WRITER_URL: https://share.opendatahub.com
+          X_ODH_AUTHENTICATION_URL: ${{ env.KEYCLOAK_URL }}/auth/
+          X_ODH_CLIENT_ID: odh-a22-dataprocessor
+          X_ODH_CLIENT_SECRET: ${{ secrets.COMBINED_CLIENT_SECRET_PROD }}
+          X_ODH_GRANT_TYPE: client_credentials
+          X_ODH_PAGINATION_SIZE: 10000
+          X_ODH_MAX_POST_BATCH_SIZE: 10000
+          X_PROVENANCE_NAME: ${{ env.PROJECT_NAME }}
+          X_PROVENANCE_VERSION: ${{ github.sha }}
+
+          X_AIRFLOW_PROJ_DIR: /opt/pollution-v2
+          X_AIRFLOW_UID: 5000
+          X_AIRFLOW__CORE__EXECUTOR: CeleryExecutor
+          X_AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
+          X_AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow
+          X_AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
+          X_AIRFLOW__CORE__FERNET_KEY: ''
+          X_AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
+          X_AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
+          X_AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session'
+          # yamllint disable rule:line-length
+          # Use simple http server on scheduler for health checks
+          # See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server
+          # yamllint enable rule:line-length
+          X_AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'
+          # WARNING: Use _PIP_ADDITIONAL_REQUIREMENTS option ONLY for a quick checks
+          # for other purpose (development, test and especially production usage) build/extend Airflow image.
+          X__PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
+          X_AIRFLOW_VAR_ODH_AUTHENTICATION_URL: ${{ env.KEYCLOAK_URL }}/auth/
+          X_AIRFLOW_VAR_ODH_BASE_READER_URL: 'https://mobility.api.opendatahub.com'
+          X_AIRFLOW_VAR_ODH_BASE_WRITER_URL: 'https://share.opendatahub.com'
+          X_AIRFLOW_VAR_ODH_CLIENT_ID: 'odh-a22-dataprocessor'
+          X_AIRFLOW_VAR_ODH_CLIENT_SECRET: ${{ secrets.COMBINED_CLIENT_SECRET_PROD }}
+          X_AIRFLOW_VAR_ODH_GRANT_TYPE: client_credentials
+          X_AIRFLOW_VAR_ODH_MAX_POST_BATCH_SIZE: '10000'
+          X_AIRFLOW_VAR_ODH_PAGINATION_SIZE: '10000'
+          X_AIRFLOW_WWW_USER_USERNAME: ${{ secrets.AIRFLOW_WWW_USER_USERNAME }}
+          X_AIRFLOW_WWW_USER_PASSWORD: ${{ secrets.AIRFLOW_WWW_USER_PASSWORD }}
+          X_AIRFLOW_VAR_DATATYPE_PREFIX: ''
+
+          X_AIRFLOW_VAR_DAG_POLLUTION_EXECUTION_CRONTAB: 0 4 * * *
+          X_AIRFLOW_VAR_DAG_VALIDATION_EXECUTION_CRONTAB: 0 2 * * *
+
+          # username/pass not needed with client_credentials
+          X_AIRFLOW_VAR_ODH_PASSWORD: 'ODH_PASSWORD'
+          X_AIRFLOW_VAR_ODH_USERNAME: 'ODH_USERNAME'
+          X_AIRFLOW_VAR_ODH_MINIMUM_STARTING_DATE: "2017-01-01"
+          X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_HOST: 'redis'
+          X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_PORT: 6379
+          X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_DB: 3
+          # X_AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG: 4
+          X_AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG: 16
+
+          # Currently no logging enabled
+          # X_AIRFLOW__CORE__REMOTE_LOGGING: '<your_choice_on_remote_logging>'
+          # X_AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER: '<your_bucket_for_remote_logging>'
+          # X_AIRFLOW__CORE__REMOTE_LOG_CONN_ID: '<your_connection_id_for_remote_logging>'
+          # X_AIRFLOW__CORE__ENCRYPT_S3_LOGS: '<your_choice_on_encrypt_logs>'
+          # X_AIRFLOW_CONN_MINIO_S3_CONN: '<your_connection_details_for_remote_logging>'
+          X_NO_PROXY: '*'
+
+
+      - name: Build and push images
+        uses: noi-techpark/github-actions/docker-build-and-push@v2
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure
+          docker-username: ${{ github.actor }}
+          docker-password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Deploy application
+        uses: noi-techpark/github-actions/docker-deploy@v2
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure/ansible
+          hosts: 'prod'
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+          docker-username: 'noi-techpark-bot'
+          docker-password: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          project-name: ${{ env.PROJECT_NAME }}

pollution_v2/.gitignore

@@ -0,0 +1,14 @@
+/roles
+/airflow
+/logs
+
+venv*
+.venv
+*.pyc
+.idea
+*.log
+celerybeat-schedule.db
+/sample_data/input/odh.db
+docker-compose.yaml.custom
+docker-compose.yaml.uh
+/sample_data/odh.db

pollution_v2/CHANGELOG.md

@@ -0,0 +1,18 @@
+<!--
+SPDX-FileCopyrightText: NOI Techpark <digital@noi.bz.it>
+
+SPDX-License-Identifier: CC0-1.0
+-->
+
+# Changelog
+
+[[_TOC_]]
+
+## Version 2024.*
+
+### 2024.02.23
+
+:rocket: New features
+* Airflow as workflow management.
+* Draft for data validation workflow.
+* Deployment on https://noi.u-hopper.com.