Close macro issues

CHANGELOG

@@ -1,10 +1,18 @@
-snowplow-unified 0.2.0 (2024-02-XX)
+snowplow-unified 0.2.1 (2024-02-XX)
 ---------------------------------------
 ## Summary
 XXX
 
 ## Features
 - Add new passthrough aggregations to the views, sessions, and users table, enabled using `snowplow__view/session/user_aggregations`
+- Reorder and add some additional context fields to derived tables (non-breaking change)
+
+## Fixes
+- Fix a bug where if you ran the package in a period with no data, and had list all events enabled, the package would error rather than complete
+
+## Under the hood
+- Prefix all macro calls with package name for easier customization
+- Use macros for grouped fields (e.g. contexts) where possible
 
 ## Upgrading
 Bump the snowplow-unified version in your `packages.yml` file.

integration_tests/.scripts/integration_test.sh

@@ -26,6 +26,10 @@ for db in ${DATABASES[@]}; do
   echo "Snowplow unified integration tests: Seeding data"
   eval "dbt seed --full-refresh --target $db" || exit 1;
 
+  echo "Snowplow unified integration tests: Try run without data"
+  eval "dbt run --full-refresh --vars '{snowplow__allow_refresh: true, snowplow__backfill_limit_days: 1, snowplow__enable_cwv: false, snowplow__start_date: 2010-01-01}' --target $db" || exit 1;
+
+
   echo "Snowplow unified integration tests: Conversions"
   eval "dbt run --full-refresh --select +snowplow_unified_conversions snowplow_unified_integration_tests.source --vars '{snowplow__allow_refresh: true, snowplow__backfill_limit_days: 220, snowplow__enable_cwv: false, snowplow__enable_conversions: true}' --target $db" || exit 1;
 

integration_tests/models/actual/snowplow_unified_sessions_actual.sql

@@ -160,6 +160,7 @@ select
 ,iab__spider_or_robot
 ,yauaa__device_name
 ,yauaa__agent_class
+,yauaa__device_brand
 ,yauaa__agent_name
 ,yauaa__agent_name_version
 ,yauaa__agent_name_version_major
@@ -172,6 +173,7 @@ select
 ,yauaa__layout_engine_version
 ,yauaa__layout_engine_version_major
 ,ua__device_family
+,ua__os_family
 ,ua__os_version
 ,ua__os_major
 ,ua__os_minor

integration_tests/models/actual/snowplow_unified_views_actual.sql

@@ -173,6 +173,7 @@ iab__primary_impact,
 iab__reason,
 iab__spider_or_robot,
 yauaa__device_name,
+yauaa__device_brand,
 yauaa__agent_class,
 yauaa__agent_name,
 yauaa__agent_name_version,
@@ -186,6 +187,7 @@ yauaa__layout_engine_name_version_major,
 yauaa__layout_engine_version,
 yauaa__layout_engine_version_major,
 ua__device_family,
+ua__os_family,
 ua__os_version,
 ua__os_major,
 ua__os_minor,

integration_tests/models/expected/snowplow_unified_sessions_expected_stg.sql

@@ -67,6 +67,7 @@ SELECT
     ,os_timezone
     ,screen_resolution
     ,yauaa__device_class
+    ,yauaa__device_brand
     ,yauaa__device_version
     ,yauaa__operating_system_version
     ,yauaa__operating_system_class
@@ -181,6 +182,7 @@ SELECT
     ,yauaa__layout_engine_name_version_major
     ,yauaa__layout_engine_version
     ,yauaa__layout_engine_version_major
+    ,ua__os_family
     ,ua__device_family
     ,ua__os_version
     ,ua__os_major

integration_tests/models/expected/snowplow_unified_views_expected_stg.sql

@@ -173,6 +173,7 @@ iab__reason,
 iab__spider_or_robot,
 yauaa__device_name,
 yauaa__agent_class,
+yauaa__device_brand,
 yauaa__agent_name,
 yauaa__agent_name_version,
 yauaa__agent_name_version_major,
@@ -185,6 +186,7 @@ yauaa__layout_engine_name_version_major,
 yauaa__layout_engine_version,
 yauaa__layout_engine_version_major,
 ua__device_family,
+ua__os_family,
 ua__os_version,
 ua__os_major,
 ua__os_minor,

macros/field_definitions/conversion_query.sql

@@ -31,7 +31,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 ,SUM(CASE WHEN {{ when_condition }} THEN coalesce({{ then_condition }}, {{ conv_object.get('default_value', 0) }}) ELSE 0 END) AS cv_{{ conv_object['name'] }}_total
 {%- endif %}
 ,MIN(CASE WHEN {{ when_condition }} THEN {{ tstamp_field }} ELSE null END) AS cv_{{ conv_object['name'] }}_first_conversion
-,CAST(MAX(CASE WHEN {{ when_condition }} THEN 1 ELSE 0 END) AS {{ type_boolean() }}) AS cv_{{ conv_object['name'] }}_converted
+,CAST(MAX(CASE WHEN {{ when_condition }} THEN 1 ELSE 0 END) AS {{ dbt.type_boolean() }}) AS cv_{{ conv_object['name'] }}_converted
 {%- else -%}
 ,coalesce(cv_{{ conv_object['name'] }}_volume, 0) as cv_{{ conv_object['name'] }}_volume
 {%- if conv_object.get('list_events', false) %}
@@ -63,7 +63,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 ,SUM(CASE WHEN {{ when_condition }} THEN coalesce({{ then_condition }}, {{ conv_object.get('default_value', 0) }}) ELSE 0 END) AS cv_{{ conv_object['name'] }}_total
 {%- endif -%}
 ,MIN(CASE WHEN {{ when_condition }} THEN {{ tstamp_field }} ELSE null END) AS cv_{{ conv_object['name'] }}_first_conversion
-,CAST(MAX(CASE WHEN {{ when_condition }} THEN 1 ELSE 0 END) AS {{ type_boolean() }}) AS cv_{{ conv_object['name'] }}_converted
+,CAST(MAX(CASE WHEN {{ when_condition }} THEN 1 ELSE 0 END) AS {{ dbt.type_boolean() }}) AS cv_{{ conv_object['name'] }}_converted
 {%- else -%}
 ,coalesce(cv_{{ conv_object['name'] }}_volume, 0) as cv_{{ conv_object['name'] }}_volume
 {%- if conv_object.get('list_events', false) %}
@@ -97,7 +97,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 ,SUM(CASE WHEN {{ when_condition }} THEN coalesce({{ then_condition }}, {{ conv_object.get('default_value', 0) }}) ELSE 0 END) AS cv_{{ conv_object['name'] }}_total
 {%- endif -%}
 ,MIN(CASE WHEN {{ when_condition }} THEN {{ tstamp_field }} ELSE null END) AS cv_{{ conv_object['name'] }}_first_conversion
-,CAST(MAX(CASE WHEN {{ when_condition }} THEN 1 ELSE 0 END) AS {{ type_boolean() }}) AS cv_{{ conv_object['name'] }}_converted
+,CAST(MAX(CASE WHEN {{ when_condition }} THEN 1 ELSE 0 END) AS {{ dbt.type_boolean() }}) AS cv_{{ conv_object['name'] }}_converted
 {%- else -%}
 ,coalesce(cv_{{ conv_object['name'] }}_volume, 0) as cv_{{ conv_object['name'] }}_volume
 {%- if conv_object.get('list_events', false) %}
@@ -128,7 +128,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 ,SUM(CASE WHEN {{ when_condition }} THEN coalesce({{ then_condition }}, {{ conv_object.get('default_value', 0) }}) ELSE 0 END) AS cv_{{ conv_object['name'] }}_total
 {%- endif -%}
 ,MIN(CASE WHEN {{ when_condition }} THEN {{ tstamp_field }} ELSE null END) AS cv_{{ conv_object['name'] }}_first_conversion
-,CAST(MAX(CASE WHEN {{ when_condition }} THEN 1 ELSE 0 END) AS {{ type_boolean() }}) AS cv_{{ conv_object['name'] }}_converted
+,CAST(MAX(CASE WHEN {{ when_condition }} THEN 1 ELSE 0 END) AS {{ dbt.type_boolean() }}) AS cv_{{ conv_object['name'] }}_converted
 {%- else -%}
 ,coalesce(cv_{{ conv_object['name'] }}_volume, 0) as cv_{{ conv_object['name'] }}_volume
 {%- if conv_object.get('list_events', false) %}
@@ -160,7 +160,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 ,SUM(CASE WHEN {{ when_condition }} THEN coalesce({{ then_condition }}, {{ conv_object.get('default_value', 0) }}) ELSE 0 END) AS cv_{{ conv_object['name'] }}_total
 {%- endif -%}
 ,MIN(CASE WHEN {{ when_condition }} THEN {{ tstamp_field }} ELSE null END) AS cv_{{ conv_object['name'] }}_first_conversion
-,CAST(MAX(CASE WHEN {{ when_condition }} THEN 1 ELSE 0 END) AS {{ type_boolean() }}) AS cv_{{ conv_object['name'] }}_converted
+,CAST(MAX(CASE WHEN {{ when_condition }} THEN 1 ELSE 0 END) AS {{ dbt.type_boolean() }}) AS cv_{{ conv_object['name'] }}_converted
 {%- else -%}
 ,coalesce(cv_{{ conv_object['name'] }}_volume, 0) as cv_{{ conv_object['name'] }}_volume
 {%- if conv_object.get('list_events', false) %}

macros/field_definitions/event_counts_string_query.sql

@@ -10,7 +10,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 {% endmacro %}
 
 {% macro default__event_counts_string_query() %}
-  {% set event_names =  dbt_utils.get_column_values(ref('snowplow_unified_events_this_run'), 'event_name', order_by = 'event_name') %}
+  {% set event_names =  dbt_utils.get_column_values(ref('snowplow_unified_events_this_run'), 'event_name', order_by = 'event_name', default = []) %}
   {# Loop over every event_name in this run, create a json string of the name and count ONLY if there are events with that name in the session (otherwise empty string),
   then trim off the last comma (cannot use loop.first/last because first/last entry may not have any events for that session)
   #}
@@ -24,7 +24,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 {% endmacro %}
 
 {% macro bigquery__event_counts_string_query() %}
-  {% set event_names =  dbt_utils.get_column_values(ref('snowplow_unified_events_this_run'), 'event_name', order_by = 'event_name') %}
+  {% set event_names =  dbt_utils.get_column_values(ref('snowplow_unified_events_this_run'), 'event_name', order_by = 'event_name', default = []) %}
   {# Loop over every event_name in this run, create a json string of the name and count ONLY if there are events with that name in the session (otherwise empty string),
   then trim off the last comma (cannot use loop.first/last because first/last entry may not have any events for that session)
   #}
@@ -39,18 +39,22 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 
 {% macro spark__event_counts_string_query() %}
 
-  {% set event_names =  dbt_utils.get_column_values(ref('snowplow_unified_base_events_this_run'), 'event_name', order_by = 'event_name') %}
+  {% set event_names =  dbt_utils.get_column_values(ref('snowplow_unified_base_events_this_run'), 'event_name', order_by = 'event_name', default = []) %}
   {# Loop over every event_name in this run, create a map of the name and count, later filter for only events with that name in the session #}
-  map(
-  {%- for event_name in event_names %}
-      '{{event_name}}',  sum(case when event_name = '{{event_name}}' then 1 else 0 end){% if not loop.last %},{% endif %}
-  {%- endfor -%}
-  )
+  {% if event_names %}
+    map(
+    {%- for event_name in event_names %}
+        '{{event_name}}',  sum(case when event_name = '{{event_name}}' then 1 else 0 end){% if not loop.last %},{% endif %}
+    {%- endfor -%}
+    )
+  {% else %}
+    cast(null as map<string, int>)
+  {% endif %}
 
 {% endmacro %}
 
 {% macro postgres__event_counts_string_query() %}
-  {% set event_names =  dbt_utils.get_column_values(ref('snowplow_unified_events_this_run'), 'event_name', order_by = 'event_name') %}
+  {% set event_names =  dbt_utils.get_column_values(ref('snowplow_unified_events_this_run'), 'event_name', order_by = 'event_name', default = []) %}
   {# Loop over every event_name in this run, create a json string of the name and count ONLY if there are events with that name in the session (otherwise empty string),
   then trim off the last comma (cannot use loop.first/last because first/last entry may not have any events for that session)
   #}

macros/field_extractions/get_app_context_fields.sql

@@ -32,8 +32,8 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
           relation=source('atomic', 'events') if 'integration_tests' not in project_name and 'snowplow' not in project_name else ref('snowplow_unified_events_stg'),
           relation_alias=none) }}
   {% else %}
-    , cast(null as {{ type_string() }}) as app__build
-    , cast(null as {{ type_string() }}) as app__version
+    , cast(null as {{ dbt.type_string() }}) as app__build
+    , cast(null as {{ dbt.type_string() }}) as app__version
   {% endif %}
 {% endmacro %}
 
@@ -42,8 +42,8 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
       , contexts_com_snowplowanalytics_mobile_application_1[0].build::STRING AS app__build
       , contexts_com_snowplowanalytics_mobile_application_1[0].version::STRING AS app__version
   {% else %}
-      , cast(null as {{ type_string() }}) as app__build
-      , cast(null as {{ type_string() }}) as app__version
+      , cast(null as {{ dbt.type_string() }}) as app__build
+      , cast(null as {{ dbt.type_string() }}) as app__version
   {% endif %}
 {% endmacro %}
 
@@ -52,7 +52,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
       , contexts_com_snowplowanalytics_mobile_application_1[0]:build::varchar(255) AS app__build
       , contexts_com_snowplowanalytics_mobile_application_1[0]:version::varchar(255) AS app__version
     {% else %}
-      , cast(null as {{ type_string() }}) as app__build
-      , cast(null as {{ type_string() }}) as app__version
+      , cast(null as {{ dbt.type_string() }}) as app__build
+      , cast(null as {{ dbt.type_string() }}) as app__version
     {% endif %}
 {% endmacro %}

macros/field_extractions/get_app_error_event_fields.sql

@@ -16,10 +16,10 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
     , cast(null as {{ snowplow_utils.type_max_string() }}) as app_error__programming_language
     , cast(null as {{ snowplow_utils.type_max_string() }}) as app_error__class_name
     , cast(null as {{ snowplow_utils.type_max_string() }}) as app_error__exception_name
-    , cast(null as {{ type_boolean() }}) as app_error__is_fatal
-    , cast(null as {{ type_int() }}) as app_error__line_number
+    , cast(null as {{ dbt.type_boolean() }}) as app_error__is_fatal
+    , cast(null as {{ dbt.type_int() }}) as app_error__line_number
     , cast(null as {{ snowplow_utils.type_max_string() }}) as app_error__stack_trace
-    , cast(null as {{ type_int() }}) as app_error__thread_id
+    , cast(null as {{ dbt.type_int() }}) as app_error__thread_id
     , cast(null as {{ snowplow_utils.type_max_string() }}) as app_error__thread_name
   {% endif %}
 {% endmacro %}
@@ -46,15 +46,15 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
           relation=source('atomic', 'events') if 'integration_tests' not in project_name and 'snowplow' not in project_name else ref('snowplow_unified_events_stg'),
           relation_alias=none) }}
   {% else %}
-    , cast(null as {{ type_string() }}) as app_error__message
-    , cast(null as {{ type_string() }}) as app_error__programming_language
-    , cast(null as {{ type_string() }}) as app_error__class_name
-    , cast(null as {{ type_string() }}) as app_error__exception_name
-    , cast(null as {{ type_boolean() }}) as app_error__is_fatal
-    , cast(null as {{ type_int() }}) as app_error__line_number
-    , cast(null as {{ type_string() }}) as app_error__stack_trace
-    , cast(null as {{ type_int() }}) as app_error__thread_id
-    , cast(null as {{ type_string() }}) as app_error__thread_name
+    , cast(null as {{ dbt.type_string() }}) as app_error__message
+    , cast(null as {{ dbt.type_string() }}) as app_error__programming_language
+    , cast(null as {{ dbt.type_string() }}) as app_error__class_name
+    , cast(null as {{ dbt.type_string() }}) as app_error__exception_name
+    , cast(null as {{ dbt.type_boolean() }}) as app_error__is_fatal
+    , cast(null as {{ dbt.type_int() }}) as app_error__line_number
+    , cast(null as {{ dbt.type_string() }}) as app_error__stack_trace
+    , cast(null as {{ dbt.type_int() }}) as app_error__thread_id
+    , cast(null as {{ dbt.type_string() }}) as app_error__thread_name
   {% endif %}
 {% endmacro %}
 
@@ -70,15 +70,15 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
     , unstruct_event_com_snowplowanalytics_snowplow_application_error_1.thread_id::INT AS app_error__thread_id
     , unstruct_event_com_snowplowanalytics_snowplow_application_error_1.thread_name::STRING AS app_error__thread_name
   {% else %}
-    , cast(null as {{ type_string() }}) as app_error__message
-    , cast(null as {{ type_string() }}) as app_error__programming_language
-    , cast(null as {{ type_string() }}) as app_error__class_name
-    , cast(null as {{ type_string() }}) as app_error__exception_name
-    , cast(null as {{ type_boolean() }}) as app_error__is_fatal
-    , cast(null as {{ type_int() }}) as app_error__line_number
-    , cast(null as {{ type_string() }}) as app_error__stack_trace
-    , cast(null as {{ type_int() }}) as app_error__thread_id
-    , cast(null as {{ type_string() }}) as app_error__thread_name
+    , cast(null as {{ dbt.type_string() }}) as app_error__message
+    , cast(null as {{ dbt.type_string() }}) as app_error__programming_language
+    , cast(null as {{ dbt.type_string() }}) as app_error__class_name
+    , cast(null as {{ dbt.type_string() }}) as app_error__exception_name
+    , cast(null as {{ dbt.type_boolean() }}) as app_error__is_fatal
+    , cast(null as {{ dbt.type_int() }}) as app_error__line_number
+    , cast(null as {{ dbt.type_string() }}) as app_error__stack_trace
+    , cast(null as {{ dbt.type_int() }}) as app_error__thread_id
+    , cast(null as {{ dbt.type_string() }}) as app_error__thread_name
   {% endif %}
 {% endmacro %}
 
@@ -94,14 +94,14 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
     , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:threadId::INT AS app_error__thread_id
     , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:threadName::VARCHAR() AS app_error__thread_name
   {% else %}
-    , cast(null as {{ type_string() }}) as app_error__message
-    , cast(null as {{ type_string() }}) as app_error__programming_language
-    , cast(null as {{ type_string() }}) as app_error__class_name
-    , cast(null as {{ type_string() }}) as app_error__exception_name
-    , cast(null as {{ type_boolean() }}) as app_error__is_fatal
-    , cast(null as {{ type_int() }}) as app_error__line_number
-    , cast(null as {{ type_string() }}) as app_error__stack_trace
-    , cast(null as {{ type_int() }}) as app_error__thread_id
-    , cast(null as {{ type_string() }}) as app_error__thread_name
+    , cast(null as {{ dbt.type_string() }}) as app_error__message
+    , cast(null as {{ dbt.type_string() }}) as app_error__programming_language
+    , cast(null as {{ dbt.type_string() }}) as app_error__class_name
+    , cast(null as {{ dbt.type_string() }}) as app_error__exception_name
+    , cast(null as {{ dbt.type_boolean() }}) as app_error__is_fatal
+    , cast(null as {{ dbt.type_int() }}) as app_error__line_number
+    , cast(null as {{ dbt.type_string() }}) as app_error__stack_trace
+    , cast(null as {{ dbt.type_int() }}) as app_error__thread_id
+    , cast(null as {{ dbt.type_string() }}) as app_error__thread_name
   {% endif %}
 {% endmacro %}