Release/snowplow unified/0.5.1

CHANGELOG

@@ -1,3 +1,19 @@
+snowplow-unified 0.5.1 (2025-01-24)
+---------------------------------------
+
+## Summary
+This release includes important updates and enhancements to improve the functionality, maintainability, and initial checks of the Snowplow Unified package.
+
+## Features
+- Rework filter bots macro: Improved logic for filtering bot events to ensure cleaner data handling and more accurate analytics.
+- Update snowplow_unified_dim_ga4_source_categories: Enhanced the dim_ga4_source_categories table to provide better support for categorization and reporting.
+Fixes
+- Remove duplicate 2T codes: Addressed an issue where duplicate 2T codes caused inconsistencies in data processing.
+## Under the hood
+- Add required fields to initial checks helper macro: Improved initialization checks by including necessary fields for robust and early error detection.
+## Upgrading
+- Update the snowplow-unified version in your packages.yml file to take advantage of these improvements.
+
 snowplow-unified 0.5.0 (2024-10-14)
 ---------------------------------------
 ## Summary

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'snowplow_unified'
-version: '0.5.0'
+version: '0.5.1'
 config-version: 2
 
 require-dbt-version: [">=1.6.0", "<2.0.0"]

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'snowplow_unified_integration_tests'
-version: '0.5.0'
+version: '0.5.1'
 config-version: 2
 
 profile: 'integration_tests'

integration_tests/models/source/bigquery/snowplow_unified_events_stg.sql

@@ -494,7 +494,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
       select as struct JSON_EXTRACT_scalar(json_array,'$.category') as category,
                     JSON_EXTRACT_scalar(json_array,'$.primary_impact') as primary_impact,
                     JSON_EXTRACT_scalar(json_array,'$.reason') as reason,
-                    JSON_EXTRACT_scalar(json_array,'$.spider_or_robot') as spider_or_robot
+                    cast(JSON_EXTRACT_scalar(json_array,'$.spider_or_robot') as boolean) as spider_or_robot
 
       from unnest(contexts_com_iab_snowplow_spiders_and_robots_1_0_0) as json_array
       ) as contexts_com_iab_snowplow_spiders_and_robots_1_0_0

macros/context_existance_check.sql

@@ -25,7 +25,10 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
           'contexts_com_snowplowanalytics_mobile_application_1' if target.type not in ['redshift', 'postgres'] else var('snowplow__application_context')
       ],
       "snowplow__enable_browser_context": [
-          'contexts_com_snowplowanalytics_snowplow_web_page_1' if target.type not in ['redshift', 'postgres'] else var('snowplow__browser_context')
+          'com_snowplowanalytics_snowplow_browser_context_1' if target.type not in ['redshift', 'postgres'] else var('snowplow__browser_context')
+      ],
+      "snowplow__enable_browser_context_2": [
+          'com_snowplowanalytics_snowplow_browser_context_2' if target.type not in ['redshift', 'postgres'] else var('snowplow__browser_context_2')
       ],
       "snowplow__enable_geolocation_context": [
           'contexts_com_snowplowanalytics_snowplow_geolocation_context_1' if target.type not in ['redshift', 'postgres'] else var('snowplow__geolocation_context')
@@ -48,7 +51,15 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
       ],
       "snowplow__enable_app_errors": [
           'unstruct_event_com_snowplowanalytics_snowplow_application_error_1' if target.type not in ['redshift', 'postgres'] else var('snowplow__application_error_events')
+      ],
+      "snowplow__enable_mobile": [
+          'contexts_com_snowplowanalytics_snowplow_client_session_1' if target.type not in ['redshift', 'postgres'] else var('snowplow__session_context'),
+          'unstruct_event_com_snowplowanalytics_mobile_screen_view_1' if target.type not in ['redshift', 'postgres'] else var('snowplow__screen_context')
+      ],
+      "snowplow__enable_web": [
+          'contexts_com_snowplowanalytics_snowplow_web_page_1' if target.type not in ['redshift', 'postgres'] else var('snowplow__page_view_context')
       ]
+
   }
 
   %}

macros/filter_bots.sql

@@ -10,17 +10,31 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 {%- endmacro -%}
 
 {% macro default__filter_bots(table_alias = none) %}
+  {% if var('snowplow__enable_iab', false) %}
+    {# additional logic in case the result is null due to server anonymization #}
+    and coalesce(iab__spider_or_robot, False ) = False
+  {% endif %}
   and {% if table_alias %}{{table_alias~'.'}}{% endif %}useragent not similar to '%(bot|crawl|slurp|spider|archiv|spinn|sniff|seo|audit|survey|pingdom|worm|capture|(browser|screen)shots|analyz|index|thumb|check|facebook|PingdomBot|PhantomJS|YandexBot|Twitterbot|a_archiver|facebookexternalhit|Bingbot|BingPreview|Googlebot|Baiduspider|360(Spider|User-agent)|semalt)%'
 {% endmacro %}
 
 {% macro bigquery__filter_bots(table_alias = none) %}
+  {% if var('snowplow__enable_iab', false) %}
+    and coalesce(iab__spider_or_robot, False ) = False
+  {% endif %}
   and not regexp_contains({% if table_alias %}{{table_alias~'.'}}{% endif %}useragent, '(bot|crawl|slurp|spider|archiv|spinn|sniff|seo|audit|survey|pingdom|worm|capture|(browser|screen)shots|analyz|index|thumb|check|facebook|PingdomBot|PhantomJS|YandexBot|Twitterbot|a_archiver|facebookexternalhit|Bingbot|BingPreview|Googlebot|Baiduspider|360(Spider|User-agent)|semalt)')
 {% endmacro %}
 
 {% macro spark__filter_bots(table_alias = none) %}
+  {% if var('snowplow__enable_iab', false) %}
+   {# had to add different syntax as the coalesce based one resulted in a Spark error #}
+    and (not iab__spider_or_robot = True or iab__spider_or_robot is null)
+  {% endif %}
   and not rlike({% if table_alias %}{{table_alias~'.'}}{% endif %}useragent, '.*(bot|crawl|slurp|spider|archiv|spinn|sniff|seo|audit|survey|pingdom|worm|capture|(browser|screen)shots|analyz|index|thumb|check|facebook|PingdomBot|PhantomJS|YandexBot|Twitterbot|a_archiver|facebookexternalhit|Bingbot|BingPreview|Googlebot|Baiduspider|360(Spider|User-agent)|semalt).*')
 {% endmacro %}
 
 {% macro snowflake__filter_bots(table_alias = none) %}
+  {% if var('snowplow__enable_iab', false) %}
+    and coalesce(iab__spider_or_robot, False ) = False
+  {% endif %}
   and not rlike({% if table_alias %}{{table_alias~'.'}}{% endif %}useragent, '.*(bot|crawl|slurp|spider|archiv|spinn|sniff|seo|audit|survey|pingdom|worm|capture|(browser|screen)shots|analyz|index|thumb|check|facebook|PingdomBot|PhantomJS|YandexBot|Twitterbot|a_archiver|facebookexternalhit|Bingbot|BingPreview|Googlebot|Baiduspider|360(Spider|User-agent)|semalt).*')
 {% endmacro %}

models/optional_modules/core_web_vitals/scratch/bigquery/snowplow_unified_web_vital_events_this_run.sql

@@ -70,14 +70,6 @@ with prep as (
 
   -- exclude bot traffic
 
-  {% if var('snowplow__enable_iab', false) %}
-    and not {{ snowplow_utils.get_field(column_name = 'contexts_com_iab_snowplow_spiders_and_robots_1_0_0',
-                              field_name = 'spider_or_robot',
-                              table_alias = 'e',
-                              type = 'boolean',
-                              array_index = 0)}} = True
-  {% endif %}
-
   {{ snowplow_unified.filter_bots() }}
 
 )

models/optional_modules/core_web_vitals/scratch/databricks/snowplow_unified_web_vital_events_this_run.sql

@@ -70,14 +70,6 @@ with prep as (
 
   -- exclude bot traffic
 
-  {% if var('snowplow__enable_iab', false) %}
-    and not {{ snowplow_utils.get_field(column_name = 'contexts_com_iab_snowplow_spiders_and_robots_1',
-                              field_name = 'spider_or_robot',
-                              table_alias = 'e',
-                              type = 'boolean',
-                              array_index = 0)}} = True
-  {% endif %}
-
   {{ snowplow_unified.filter_bots() }}
 
 )

models/optional_modules/core_web_vitals/scratch/default/snowplow_unified_web_vital_events_this_run.sql

@@ -69,10 +69,6 @@ with prep as (
 
   -- exclude bot traffic
 
-  {% if var('snowplow__enable_iab', false) %}
-    and not e.iab__spider_or_robot = True
-  {% endif %}
-
   {{ snowplow_unified.filter_bots() }}
 
 )

seeds/seeds.yml

@@ -29,6 +29,9 @@ seeds:
     description: A mapping between the ISO 639-1 and ISO 639-2/T language code to the full name of the language. Source https://www.loc.gov/standards/iso639-2/php/code_list.php
     columns:
       - name: iso_639_2t_code
+        data_tests:
+          - unique
+          - not_null
       - name: iso_639_1_code
       - name: name
   - name: snowplow_unified_dim_iso_639_3