From a81cf8732bf940d23ecbd535ee811bb5ee423b35 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 5 Apr 2022 10:56:39 -0500 Subject: [PATCH 1/7] Update stg_fivetran_log__log.sql --- models/staging/stg_fivetran_log__log.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/staging/stg_fivetran_log__log.sql b/models/staging/stg_fivetran_log__log.sql index 972dc9d1..7ac0f24c 100644 --- a/models/staging/stg_fivetran_log__log.sql +++ b/models/staging/stg_fivetran_log__log.sql @@ -22,4 +22,6 @@ fields as ( from log ) -select * from fields \ No newline at end of file +select * +from fields +where transformation_id is null From 1f6dbd9fd771544abbdb853cc2506717e8ffa134 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Wed, 13 Apr 2022 08:58:53 -0500 Subject: [PATCH 2/7] Update fivetran_log.yml --- models/fivetran_log.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/fivetran_log.yml b/models/fivetran_log.yml index 6bc7ffa6..db321b4f 100644 --- a/models/fivetran_log.yml +++ b/models/fivetran_log.yml @@ -290,6 +290,7 @@ models: combination_of_columns: - connector_id - destination_id + - message_data - created_at columns: - name: connector_id @@ -310,4 +311,4 @@ models: - name: table_name description: Name of the table, if a table is being altered or created. - name: schema_name - description: Name of the schema, if a schema or table is being created. \ No newline at end of file + description: Name of the schema, if a schema or table is being created. From 27314e9f316884a9ec2a7da8d0ca343c43a07c3b Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Wed, 13 Apr 2022 09:10:12 -0500 Subject: [PATCH 3/7] Update sample.profiles.yml --- integration_tests/ci/sample.profiles.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 51e23f5b..c85452c5 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -47,7 +47,7 @@ integration_tests: spark: type: spark method: http - schema: fivetran_log_integration__test + schema: fivetranlog_integration__test host: "{{ env_var('CI_SPARK_DBT_HOST') }}" organization: "{{ env_var('CI_SPARK_DBT_ORGANIZATION') }}" token: "{{ env_var('CI_SPARK_DBT_TOKEN') }}" From 2b419d5d92d626e16fd87dda200b766ee1459d67 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Mon, 18 Apr 2022 12:59:54 -0500 Subject: [PATCH 4/7] updates to the connector status model and update to the changelog schema test --- models/fivetran_log__connector_status.sql | 7 ++++++- models/staging/stg_fivetran_log__log.sql | 3 +-- models/staging/stg_fivetran_log__transformation.sql | 3 +-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/models/fivetran_log__connector_status.sql b/models/fivetran_log__connector_status.sql index ccc58412..d5f11bdd 100644 --- a/models/fivetran_log__connector_status.sql +++ b/models/fivetran_log__connector_status.sql @@ -1,10 +1,15 @@ -with connector_log as ( +with transformation_removal as ( select *, sum( case when event_subtype in ('sync_start') then 1 else 0 end) over ( partition by connector_id order by created_at rows unbounded preceding) as sync_batch_id from {{ ref('stg_fivetran_log__log') }} + where event_subtype != 'TRANSFORMATION' +), +connector_log as ( + select * + from transformation_removal -- only looking at errors, warnings, and syncs here where event_type = 'SEVERE' or event_type = 'WARNING' diff --git a/models/staging/stg_fivetran_log__log.sql b/models/staging/stg_fivetran_log__log.sql index 7ac0f24c..c7c7402c 100644 --- a/models/staging/stg_fivetran_log__log.sql +++ b/models/staging/stg_fivetran_log__log.sql @@ -23,5 +23,4 @@ fields as ( ) select * -from fields -where transformation_id is null +from fields \ No newline at end of file diff --git a/models/staging/stg_fivetran_log__transformation.sql b/models/staging/stg_fivetran_log__transformation.sql index f5235838..4c91bee4 100644 --- a/models/staging/stg_fivetran_log__transformation.sql +++ b/models/staging/stg_fivetran_log__transformation.sql @@ -24,5 +24,4 @@ fields as ( ) -select * from fields -where transformation_id is not null \ No newline at end of file +select * from fields \ No newline at end of file From 6830e7182826e40d82208fb030f24913484404e3 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Mon, 18 Apr 2022 14:29:22 -0500 Subject: [PATCH 5/7] changelog and versioning upgrades --- CHANGELOG.md | 4 ++++ dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- models/fivetran_log__connector_status.sql | 10 ++++++---- models/staging/stg_fivetran_log__transformation.sql | 3 ++- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3166f13..23246719 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# dbt_fivetran_log v0.5.4 +## Fixes +- The unique combination of columns test within the `fivetran_log__schema_changelog` model has been updated to also check the `message_data` field. This is needed as schema changelog events may now sync at the same time. ([#51](https://github.com/fivetran/dbt_fivetran_log/pull/51)) +- The `fivetran_log__connector_status` model has been adjusted to filter out all `TRANSFORMATION` event_subtype logs. Transformation logs are not always synced as a JSON object and thus the package may encounter errors on Snowflake warehouses when parsing non-JSON fields. Since transformation records are not used in this end model, they have been filtered out. ([#51](https://github.com/fivetran/dbt_fivetran_log/pull/51)) # dbt_fivetran_log v0.5.3 ## Fixes - Per the [Fivetran Log December 2021 Release Notes](https://fivetran.com/docs/logs/changelog#december2021) every sync results in a final `sync_end` event. In the previous version of this package, a successful sync was identified via a `sync_end` event while anything else was a version of broken. Since all syncs result in a `sync_end` event now, the package has been updated to account for this change within the connector. diff --git a/dbt_project.yml b/dbt_project.yml index 6e6d92fd..4504d519 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'fivetran_log' -version: '0.5.3' +version: '0.5.4' require-dbt-version: [">=1.0.0", "<2.0.0"] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index ef86a60b..8b60c7b2 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'fivetran_log_integration_tests' -version: '0.5.3' +version: '0.5.4' config-version: 2 profile: 'integration_tests' diff --git a/models/fivetran_log__connector_status.sql b/models/fivetran_log__connector_status.sql index d5f11bdd..36e81ed9 100644 --- a/models/fivetran_log__connector_status.sql +++ b/models/fivetran_log__connector_status.sql @@ -1,14 +1,16 @@ with transformation_removal as ( - select *, - sum( case when event_subtype in ('sync_start') then 1 else 0 end) over ( partition by connector_id - order by created_at rows unbounded preceding) as sync_batch_id + select * from {{ ref('stg_fivetran_log__log') }} where event_subtype != 'TRANSFORMATION' + ), connector_log as ( - select * + select + *, + sum( case when event_subtype in ('sync_start') then 1 else 0 end) over ( partition by connector_id + order by created_at rows unbounded preceding) as sync_batch_id from transformation_removal -- only looking at errors, warnings, and syncs here where event_type = 'SEVERE' diff --git a/models/staging/stg_fivetran_log__transformation.sql b/models/staging/stg_fivetran_log__transformation.sql index 4c91bee4..9427b7d0 100644 --- a/models/staging/stg_fivetran_log__transformation.sql +++ b/models/staging/stg_fivetran_log__transformation.sql @@ -24,4 +24,5 @@ fields as ( ) -select * from fields \ No newline at end of file +select * +from fields \ No newline at end of file From 3f409ad29b5248f8df490dad3523b7dd558d7695 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Mon, 18 Apr 2022 14:38:01 -0500 Subject: [PATCH 6/7] formatting fixes --- models/staging/stg_fivetran_log__log.sql | 3 +-- models/staging/stg_fivetran_log__transformation.sql | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/models/staging/stg_fivetran_log__log.sql b/models/staging/stg_fivetran_log__log.sql index c7c7402c..972dc9d1 100644 --- a/models/staging/stg_fivetran_log__log.sql +++ b/models/staging/stg_fivetran_log__log.sql @@ -22,5 +22,4 @@ fields as ( from log ) -select * -from fields \ No newline at end of file +select * from fields \ No newline at end of file diff --git a/models/staging/stg_fivetran_log__transformation.sql b/models/staging/stg_fivetran_log__transformation.sql index 9427b7d0..f5235838 100644 --- a/models/staging/stg_fivetran_log__transformation.sql +++ b/models/staging/stg_fivetran_log__transformation.sql @@ -24,5 +24,5 @@ fields as ( ) -select * -from fields \ No newline at end of file +select * from fields +where transformation_id is not null \ No newline at end of file From b6eda29761c43cbc891f2485392583912df4c859 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Mon, 18 Apr 2022 15:09:53 -0500 Subject: [PATCH 7/7] using transformation_id instead of event_subtype` --- CHANGELOG.md | 2 +- models/fivetran_log__connector_status.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23246719..e8b95117 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # dbt_fivetran_log v0.5.4 ## Fixes - The unique combination of columns test within the `fivetran_log__schema_changelog` model has been updated to also check the `message_data` field. This is needed as schema changelog events may now sync at the same time. ([#51](https://github.com/fivetran/dbt_fivetran_log/pull/51)) -- The `fivetran_log__connector_status` model has been adjusted to filter out all `TRANSFORMATION` event_subtype logs. Transformation logs are not always synced as a JSON object and thus the package may encounter errors on Snowflake warehouses when parsing non-JSON fields. Since transformation records are not used in this end model, they have been filtered out. ([#51](https://github.com/fivetran/dbt_fivetran_log/pull/51)) +- The `fivetran_log__connector_status` model has been adjusted to filter out all logs that contain a `transformation_id`. Transformation logs are not always synced as a JSON object and thus the package may encounter errors on Snowflake warehouses when parsing non-JSON fields. Since transformation records are not used in this end model, they have been filtered out. ([#51](https://github.com/fivetran/dbt_fivetran_log/pull/51)) # dbt_fivetran_log v0.5.3 ## Fixes - Per the [Fivetran Log December 2021 Release Notes](https://fivetran.com/docs/logs/changelog#december2021) every sync results in a final `sync_end` event. In the previous version of this package, a successful sync was identified via a `sync_end` event while anything else was a version of broken. Since all syncs result in a `sync_end` event now, the package has been updated to account for this change within the connector. diff --git a/models/fivetran_log__connector_status.sql b/models/fivetran_log__connector_status.sql index 36e81ed9..e859139f 100644 --- a/models/fivetran_log__connector_status.sql +++ b/models/fivetran_log__connector_status.sql @@ -2,7 +2,7 @@ with transformation_removal as ( select * from {{ ref('stg_fivetran_log__log') }} - where event_subtype != 'TRANSFORMATION' + where transformation_id is null ),