Adjust transactions variable

NASA-IMPACT · Feb 13, 2025 · ad3fac7 · ad3fac7
1 parent 6eb3d05
commit ad3fac7
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 5 deletions.
diff --git a/dags/veda_data_pipeline/groups/processing_tasks.py b/dags/veda_data_pipeline/groups/processing_tasks.py
@@ -1,4 +1,3 @@
-from datetime import timedelta
 import json
 import logging
 from copy import deepcopy
@@ -10,6 +9,9 @@
 
 group_kwgs = {"group_id": "Process", "tooltip": "Process"}
 
+airflow_vars = Variable.get("aws_dags_variables")
+airflow_vars_json = json.loads(airflow_vars)
+TRANSACTIONS_ENDPOINT_ENABLED = airflow_vars_json.get("TRANSACTIONS_ENDPOINT_ENABLED", False)
 
 def log_task(text: str):
     logging.info(text)
@@ -30,9 +32,8 @@ def remove_thumbnail_asset(ti):
         payload.pop("assets")
     return payload
 
-TRANSACTIONS_ENDPOINT_ENABLED = False
 if TRANSACTIONS_ENDPOINT_ENABLED:
-    # assuming default chunk size, this matches the current dynamoDB configuration on the STAC ingestor
+    # assuming default chunk size (500), this matches the current dynamoDB configuration on the STAC ingestor
     task_kwargs = {"retries": 3, "retry_delay": 10, "retry_exponential_backoff": True, "max_active_tis_per_dag": 2}
     submit_handler = submit_transactions_handler
 else:

diff --git a/dags/veda_data_pipeline/veda_promotion_pipeline.py b/dags/veda_data_pipeline/veda_promotion_pipeline.py
@@ -86,8 +86,6 @@ def transfer_assets_to_production_bucket(ti=None, payload={}):
     return payload
 
 with DAG("veda_promotion_pipeline", params=template_dag_run_conf, **dag_args) as dag:
-    # ECS dependency variable
-
     start = EmptyOperator(task_id="start", dag=dag)
     end = EmptyOperator(task_id="end", dag=dag)