Merge pull request #64 from get-select/dev

Increase incremental lookback window to account for changing cost data
get-select · Jan 7, 2023 · b8fa77a · b8fa77a
2 parents 4166fc7 + af04834
commit b8fa77a
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 3 deletions.
diff --git a/.changes/unreleased/Fixes-20230106-215452.yaml b/.changes/unreleased/Fixes-20230106-215452.yaml
@@ -0,0 +1,6 @@
+kind: Fixes
+body: Increase incremental lookback window to account for changing cost data
+time: 2023-01-06T21:54:52.198505-07:00
+custom:
+  Author: ian-whitestone
+  PR: "64"
diff --git a/models/query_history_enriched.sql b/models/query_history_enriched.sql
@@ -1,4 +1,4 @@
-{{ config(materialized='incremental') }}
+{{ config(materialized='incremental', unique_key=['query_id', 'start_time']) }}
 
 with
 query_history as (
@@ -16,15 +16,19 @@ query_history as (
     from {{ ref('stg_query_history') }}
 
     {% if is_incremental() %}
-        where end_time > (select max(end_time) from {{ this }})
+        -- Conservatively re-process the last 7 days to account for late arriving rates data
+        -- which changes the cost per query
+        where end_time > (select dateadd(day, -7, max(end_time)) from {{ this }})
     {% endif %}
 ),
 
 cost_per_query as (
     select *
     from {{ ref('cost_per_query') }}
     {% if is_incremental() %}
-        where end_time > (select max(end_time) from {{ this }})
+        -- Conservatively re-process the last 7 days to account for late arriving rates data
+        -- which changes the cost per query
+        where end_time > (select dateadd(day, -7, max(end_time)) from {{ this }})
     {% endif %}
 )