chore: upgrade python packages in functions and set runtime to python…

… 3.11 - Firebase functions runtime to Node 20 (#736)

.github/workflows/datasets-batch-deployer.yml

@@ -39,6 +39,9 @@ on:
         description: Service account key
         required: true
 
+env:
+  python_version: '3.11'
+
 jobs:
   terraform:
     name: 'Terraform'
@@ -80,6 +83,10 @@ jobs:
           docker compose --env-file ./config/.env.local up -d postgres
         working-directory: ${{ github.workspace }}
 
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.python_version }}
+
       - name: Install Liquibase
         run: |    
           wget -O- https://repo.liquibase.com/liquibase.asc | gpg --dearmor > liquibase-keyring.gpg && \

functions-python/batch_datasets/main_local_debug.py

@@ -0,0 +1,28 @@
+# Code to be able to debug locally without affecting the runtime cloud function
+
+#
+# Requirements:
+# - Google Cloud SDK installed
+# - Make sure to have the following environment variables set in your .env.local file
+# - Local database in running state
+# - Pub/Sub emulator running
+#   - gcloud beta emulators pubsub start --project=project-id --host-port='localhost:8043'
+# - Google Datastore emulator running
+#   - gcloud beta emulators datastore start --project=project-id --host-port='localhost:8042'
+
+# Usage:
+# - python batch_datasets/main_local_debug.py
+from src.main import batch_datasets
+from dotenv import load_dotenv
+
+# Load environment variables from .env.local
+load_dotenv(dotenv_path=".env.local")
+
+if __name__ == "__main__":
+
+    class RequestObject:
+        def __init__(self, headers):
+            self.headers = headers
+
+    request = RequestObject({"X-Cloud-Trace-Context": "1234567890abcdef"})
+    batch_datasets(request)

functions-python/batch_datasets/requirements.txt

@@ -1,15 +1,20 @@
+# Common packages
 functions-framework==3.*
-google-cloud-pubsub
 google-cloud-logging
-google-cloud-datastore
 psycopg2-binary==2.9.6
-aiohttp~=3.8.6
+aiohttp~=3.10.5
 asyncio~=3.4.3
-urllib3~=2.1.0
+urllib3~=2.2.2
+requests~=2.32.3
+attrs~=23.1.0
+pluggy~=1.3.0
+certifi~=2024.7.4
+
+# SQL Alchemy and Geo Alchemy
 SQLAlchemy==2.0.23
 geoalchemy2==0.14.7
-requests~=2.31.0
+
+# Google specific packages for this function
+google-cloud-pubsub
+google-cloud-datastore
 cloudevents~=1.10.1
-attrs~=23.1.0
-pluggy~=1.3.0
-certifi~=2023.7.22

functions-python/batch_datasets/src/main.py

@@ -137,7 +137,7 @@ def batch_datasets(request):
         print(f"Publishing {data_str} to {topic_path}.")
         future = publish(publisher, topic_path, data_str.encode("utf-8"))
         future.add_done_callback(
-            lambda _: publish_callback(future, feed["stable_id"], topic_path)
+            lambda _: publish_callback(future, feed.stable_id, topic_path)
         )
     BatchExecutionService().save(
         BatchExecution(

functions-python/batch_process_dataset/.env.rename_me

@@ -0,0 +1,9 @@
+# Environment variables for tokens function to run locally. Delete this line after rename the file.
+FEEDS_DATABASE_URL=postgresql://postgres:postgres@localhost:5432/MobilityDatabase
+PROJECT_ID=my-project-id
+DATASTORE_DATASET=my-project-id
+DATASTORE_EMULATOR_HOST=localhost:8044
+DATASTORE_EMULATOR_HOST_PATH=localhost:8044/datastore
+DATASTORE_HOST=http://localhost:8044
+DATASTORE_PROJECT_ID=my-project-id
+

functions-python/batch_process_dataset/main_local_debug.py

@@ -0,0 +1,51 @@
+# Code to be able to debug locally without affecting the runtime cloud function
+import base64
+import json
+
+from cloudevents.http import CloudEvent
+
+#
+# Requirements:
+# - Google Cloud SDK installed
+# - Make sure to have the following environment variables set in your .env.local file
+# - Local database in running state
+# - Pub/Sub emulator running
+#   - gcloud beta emulators pubsub start --project=project-id --host-port='localhost:8043'
+# - Google Datastore emulator running
+#   - gcloud beta emulators datastore start --project=project-id --host-port='localhost:8042' --no-store-on-disk
+
+# Usage:
+# - python batch_process_dataset/main_local_debug.py
+
+from dotenv import load_dotenv
+from batch_process_dataset.src.main import process_dataset
+
+# Load environment variables from .env.local
+load_dotenv(dotenv_path=".env.local")
+
+if __name__ == "__main__":
+    attributes = {
+        "type": "com.google.cloud.pubsub.topic.publish",
+        "source": "//pubsub.googleapis.com/projects/sample-project/topics/sample-topic",
+    }
+    data = {
+        "message": {
+            "data": base64.b64encode(
+                json.dumps(
+                    {
+                        "execution_id": "execution_id",
+                        "producer_url": "producer_url",
+                        "feed_stable_id": "feed_stable_id",
+                        "feed_id": "feed_id",
+                        "dataset_id": "dataset_id",
+                        "dataset_hash": "dataset_hash",
+                        "authentication_type": 0,
+                        "authentication_info_url": "authentication_info_url",
+                        "api_key_parameter_name": "api_key_parameter_name",
+                    }
+                ).encode("utf-8")
+            ).decode("utf-8")
+        }
+    }
+    cloud_event = CloudEvent(attributes, data)
+    process_dataset(cloud_event)

functions-python/batch_process_dataset/requirements.txt

@@ -1,21 +1,24 @@
+# Common packages
 functions-framework==3.*
+google-cloud-logging
+psycopg2-binary==2.9.6
+aiohttp~=3.10.5
+asyncio~=3.4.3
+urllib3~=2.2.2
+requests~=2.32.3
+attrs~=23.1.0
+pluggy~=1.3.0
+certifi~=2024.7.4
+
+# SQL Alchemy and Geo Alchemy
+SQLAlchemy==2.0.23
+geoalchemy2==0.14.7
+
+# Google specific packages for this function
 google-cloud-storage
 google-cloud-pubsub
-google-cloud-logging
 google-api-core
 google-cloud-firestore
 google-cloud-datastore
 google-cloud-bigquery
-psycopg2-binary==2.9.6
-aiohttp
-asyncio
-urllib3~=2.1.0
-SQLAlchemy==2.0.23
-geoalchemy2==0.14.7
-requests~=2.31.0
-cloudevents~=1.10.1
-requests_mock
-Faker
-pytest~=7.4.3
-urllib3-mock
-requests-mock
+cloudevents~=1.10.1

functions-python/batch_process_dataset/requirements_dev.txt

@@ -1,4 +1,5 @@
 Faker
 pytest~=7.4.3
 urllib3-mock
-requests-mock
+requests-mock
+python-dotenv~=1.0.0

functions-python/big_query_ingestion/requirements.txt

@@ -1,15 +1,22 @@
+# Common packages
 functions-framework==3.*
 google-cloud-logging
-google-cloud-bigquery
-google-cloud-storage
 psycopg2-binary==2.9.6
-aiohttp~=3.8.6
+aiohttp~=3.10.5
 asyncio~=3.4.3
-urllib3~=2.1.0
-SQLAlchemy==2.0.23
-geoalchemy2==0.14.7
-requests~=2.31.0
+urllib3~=2.2.2
+requests~=2.32.3
 attrs~=23.1.0
 pluggy~=1.3.0
-certifi~=2023.7.22
+certifi~=2024.7.4
+
+# SQL Alchemy and Geo Alchemy
+SQLAlchemy==2.0.23
+geoalchemy2==0.14.7
+
+# Google specific packages for this function
+google-cloud-bigquery
+google-cloud-storage
+
+# Additional packages for this function
 pandas

functions-python/extract_location/requirements.txt

@@ -1,17 +1,24 @@
+# Common packages
 functions-framework==3.*
-google-cloud-pubsub
 google-cloud-logging
-google-cloud-datastore
 psycopg2-binary==2.9.6
-aiohttp~=3.8.6
+aiohttp~=3.10.5
 asyncio~=3.4.3
-urllib3~=2.1.0
+urllib3~=2.2.2
+requests~=2.32.3
+attrs~=23.1.0
+pluggy~=1.3.0
+certifi~=2024.7.4
+
+# SQL Alchemy and Geo Alchemy
 SQLAlchemy==2.0.23
 geoalchemy2==0.14.7
-requests~=2.31.0
+
+# Google specific packages for this function
+google-cloud-pubsub
+google-cloud-datastore
 cloudevents~=1.10.1
-attrs~=23.1.0
-pluggy~=1.3.0
-certifi~=2023.7.22
+
+# Additional packages for this function
 gtfs-kit
 pandas

functions-python/gbfs_validator/requirements.txt

@@ -1,15 +1,23 @@
+# Common packages
 functions-framework==3.*
+google-cloud-logging
+psycopg2-binary==2.9.6
+aiohttp~=3.10.5
+asyncio~=3.4.3
+urllib3~=2.2.2
+requests~=2.32.3
+attrs~=23.1.0
+pluggy~=1.3.0
+certifi~=2024.7.4
+
+# SQL Alchemy and Geo Alchemy
+SQLAlchemy==2.0.23
+geoalchemy2==0.14.7
+
+# Google specific packages for this function
 google-cloud-storage
 google-cloud-pubsub
-google-cloud-logging
 google-api-core
 google-cloud-firestore
 google-cloud-datastore
-psycopg2-binary==2.9.6
-aiohttp
-asyncio
-urllib3~=2.1.0
-SQLAlchemy==2.0.23
-geoalchemy2==0.14.7
-requests~=2.31.0
 cloudevents~=1.10.1

functions-python/preprocessed_analytics/requirements.txt

@@ -1,15 +1,22 @@
+# Common packages
 functions-framework==3.*
 google-cloud-logging
-google-cloud-bigquery
-google-cloud-storage
 psycopg2-binary==2.9.6
-aiohttp~=3.8.6
+aiohttp~=3.10.5
 asyncio~=3.4.3
-urllib3~=2.1.0
-SQLAlchemy==2.0.23
-geoalchemy2==0.14.7
-requests~=2.31.0
+urllib3~=2.2.2
+requests~=2.32.3
 attrs~=23.1.0
 pluggy~=1.3.0
-certifi~=2023.7.22
+certifi~=2024.7.4
+
+# SQL Alchemy and Geo Alchemy
+SQLAlchemy==2.0.23
+geoalchemy2==0.14.7
+
+# Google specific packages for this function
+google-cloud-bigquery
+google-cloud-storage
+
+# Additional packages for this function
 pandas

functions-python/update_validation_report/requirements.txt

@@ -1,15 +1,20 @@
+# Common packages
 functions-framework==3.*
 google-cloud-logging
-google-cloud-storage
-google-cloud-workflows
 psycopg2-binary==2.9.6
-aiohttp~=3.8.6
+aiohttp~=3.10.5
 asyncio~=3.4.3
-urllib3~=2.1.0
+urllib3~=2.2.2
+requests~=2.32.3
+attrs~=23.1.0
+pluggy~=1.3.0
+certifi~=2024.7.4
+
+# SQL Alchemy and Geo Alchemy
 SQLAlchemy==2.0.23
 geoalchemy2==0.14.7
-requests~=2.31.0
+
+# Google specific packages for this function
 cloudevents~=1.10.1
-attrs~=23.1.0
-pluggy~=1.3.0
-certifi~=2023.7.22
+google-cloud-storage
+google-cloud-workflows

functions-python/validation_report_processor/requirements.txt

@@ -1,13 +1,18 @@
+# Common packages
 functions-framework==3.*
 google-cloud-logging
 psycopg2-binary==2.9.6
-aiohttp~=3.8.6
+aiohttp~=3.10.5
 asyncio~=3.4.3
-urllib3~=2.1.0
-SQLAlchemy==2.0.23
-geoalchemy2==0.14.7
-requests~=2.31.0
-cloudevents~=1.10.1
+urllib3~=2.2.2
+requests~=2.32.3
 attrs~=23.1.0
 pluggy~=1.3.0
-certifi~=2023.7.22
+certifi~=2024.7.4
+
+# SQL Alchemy and Geo Alchemy
+SQLAlchemy==2.0.23
+geoalchemy2==0.14.7
+
+# Google specific packages for this function
+cloudevents~=1.10.1