From 1820103325bb827f29e696ea2a0bda9b8cbbb0ec Mon Sep 17 00:00:00 2001 From: Joel Labes Date: Fri, 16 Jul 2021 17:46:38 +1200 Subject: [PATCH 1/7] Update fivetran_utils dependency Resolves #19 --- packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages.yml b/packages.yml index f70da7ea..14c292e0 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - package: fivetran/fivetran_utils - version: [">=0.1.0", "<0.2.0"] + version: [">=0.2.0", "<0.3.0"] From aa7a77c1993673a40eb0a9f636486180942894a5 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Fri, 16 Jul 2021 08:58:21 -0500 Subject: [PATCH 2/7] dbt 20 version change --- dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 55755974..f84fc1c7 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'fivetran_log' -version: '0.3.1' +version: '0.4.0' require-dbt-version: [">=0.18.0", "<0.20.0"] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 7515dbaa..7347584c 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'fivetran_log_integration_tests' -version: '0.3.1' +version: '0.4.0' config-version: 2 profile: 'integration_tests' From b738e03d6557cd091f3cc262264c6d2b6f6b8425 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Fri, 16 Jul 2021 09:07:25 -0500 Subject: [PATCH 3/7] minor dbt 20 edits --- README.md | 11 +++++++++++ dbt_project.yml | 2 +- integration_tests/requirements.txt | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bc6db274..08519607 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![Apache License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![dbt Logo and Version](https://img.shields.io/static/v1?logo=dbt&label=dbt-version&message=0.20.x&color=orange) # Fivetran Log ([docs](https://fivetran.github.io/dbt_fivetran_log/#!/overview)) This package models Fivetran Log data from [our free internal connector](https://fivetran.com/docs/logs/fivetran-log). It uses account-level data in the format described by [this ERD](https://fivetran.com/docs/logs/fivetran-log#schemainformation). @@ -33,8 +34,18 @@ The package's main goals are to: ## Installation Instructions +`dbt_fivetran_log` currently supports `dbt 0.20.x`. + Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions, or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. +Include in your `packages.yml` + +```yaml +packages: + - package: fivetran/fivetran_log + version: [">=0.4.0", "<0.5.0"] +``` + ## Configuration By default, this package will run using your target database and the `fivetran_log` schema. If this is not where your Fivetran Log data is, add the following configuration to your `dbt_project.yml` file: diff --git a/dbt_project.yml b/dbt_project.yml index f84fc1c7..ceb1ec44 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -3,7 +3,7 @@ config-version: 2 name: 'fivetran_log' version: '0.4.0' -require-dbt-version: [">=0.18.0", "<0.20.0"] +require-dbt-version: ">=0.20.0" models: fivetran_log: diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 38e8ae6c..1b85aa4f 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1 +1 @@ -dbt==0.19.0 \ No newline at end of file +dbt~=0.20.0 \ No newline at end of file From f7b171d6852d3b1a607cab9b7320efe5826df896 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Fri, 16 Jul 2021 16:59:37 -0500 Subject: [PATCH 4/7] minor dbt 20 edits --- models/fivetran_log__connector_status.sql | 13 +++++++------ packages.yml | 8 ++++++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/models/fivetran_log__connector_status.sql b/models/fivetran_log__connector_status.sql index 9ed96e42..8546341e 100644 --- a/models/fivetran_log__connector_status.sql +++ b/models/fivetran_log__connector_status.sql @@ -118,18 +118,19 @@ final as ( connector_recent_logs.last_sync_started_at, connector_recent_logs.last_sync_completed_at, connector_recent_logs.set_up_at, - coalesce(schema_changes.number_of_schema_changes_last_month, 0) as number_of_schema_changes_last_month, - - {{ fivetran_utils.string_agg("case when connector_recent_logs.event_type = 'SEVERE' then connector_recent_logs.message_data else null end", "'\\n'") }} as errors_since_last_completed_sync, - {{ fivetran_utils.string_agg("case when connector_recent_logs.event_type = 'WARNING' then connector_recent_logs.message_data else null end", "'\\n'") }} as warnings_since_last_completed_sync + coalesce(schema_changes.number_of_schema_changes_last_month, 0) as number_of_schema_changes_last_month + {% if var('fivetran_log_using_sync_alert_messages', true) %} + , {{ fivetran_utils.string_agg("distinct case when connector_recent_logs.event_type = 'SEVERE' then connector_recent_logs.message_data else null end", "'\\n'") }} as errors_since_last_completed_sync + , {{ fivetran_utils.string_agg("distinct case when connector_recent_logs.event_type = 'WARNING' then connector_recent_logs.message_data else null end", "'\\n'") }} as warnings_since_last_completed_sync + {% endif %} from connector_recent_logs left join schema_changes on connector_recent_logs.connector_id = schema_changes.connector_id join destination on destination.destination_id = connector_recent_logs.destination_id - group by 1,2,3,4,5,6,7,8,9,10 + {{ dbt_utils.group_by(n=10) }} ) - +{{ log(target.type,info=true)}} select * from final \ No newline at end of file diff --git a/packages.yml b/packages.yml index 14c292e0..223a6b82 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,7 @@ packages: -- package: fivetran/fivetran_utils - version: [">=0.2.0", "<0.3.0"] +# - package: fivetran/fivetran_utils +# version: [">=0.2.0", "<0.3.0"] + + - git: https://github.com/fivetran/dbt_fivetran_utils.git + revision: release/v0.2.2-updates + warn-unpinned: false \ No newline at end of file From a6774a7f397bba934ffd83536d7291d05c7dc35f Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Fri, 16 Jul 2021 17:08:44 -0500 Subject: [PATCH 5/7] fivetran_utils hub switch --- packages.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages.yml b/packages.yml index 223a6b82..31ec803e 100644 --- a/packages.yml +++ b/packages.yml @@ -1,7 +1,3 @@ packages: -# - package: fivetran/fivetran_utils -# version: [">=0.2.0", "<0.3.0"] - - - git: https://github.com/fivetran/dbt_fivetran_utils.git - revision: release/v0.2.2-updates - warn-unpinned: false \ No newline at end of file +- package: fivetran/fivetran_utils + version: [">=0.2.0", "<0.3.0"] \ No newline at end of file From 52b96bfe18fd8ded9f4da2fd52fc756d3bc90e51 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Fri, 16 Jul 2021 18:17:23 -0500 Subject: [PATCH 6/7] dbt log removal --- models/fivetran_log__connector_status.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/fivetran_log__connector_status.sql b/models/fivetran_log__connector_status.sql index 8546341e..41e8b5b6 100644 --- a/models/fivetran_log__connector_status.sql +++ b/models/fivetran_log__connector_status.sql @@ -132,5 +132,5 @@ final as ( join destination on destination.destination_id = connector_recent_logs.destination_id {{ dbt_utils.group_by(n=10) }} ) -{{ log(target.type,info=true)}} + select * from final \ No newline at end of file From d90039c8ed95254137e0b8f9850135a97c96842a Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Mon, 19 Jul 2021 11:39:43 -0500 Subject: [PATCH 7/7] error and warnings documentation and touch up --- README.md | 15 ++++++++++++++- models/fivetran_log__connector_status.sql | 4 ++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 08519607..ea035af1 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ vars: ``` ### Disabling Transformation Models -If you have never created Fivetran-orchestrated [basic SQL transformations](https://fivetran.com/docs/transformations/basic-sql), your source data will not contain the `transformation` and `trigger_table` tables. Moreover, if you have only created *scheduled* basic transformations that are not triggered by table syncs, your source data will not contain the `trigger_table` table (though it will contain `transformation`). +If you have never created Fivetran-orchestrated [basic SQL transformations](https://fivetran.com/docs/transformations/basic-sql), your source data will not contain the `transformation` and `trigger_table` tables. Moreover, if you have only created *scheduled* basic transformations that are not triggered by table syncs, your source data will not contain the `trigger_table` table (though it will contain `transformation`). To disable the corresponding functionality in the package, you must add the following variable(s) to your `dbt_project.yml` file. By default, all variables are assumed to be `true`: @@ -78,6 +78,19 @@ vars: fivetran_log_using_triggers: false # this will disable only trigger_table logic ``` +### Disabling Fivetran Error and Warning Messages +Some users may wish to exclude Fivetran error and warnings messages from the final `fivetran_log__connector_status` model due to the length of the message. To disable the `errors_since_last_completed_sync` and `warnings_since_last_completed_sync` fields from the final model you may add the following variable to you to your `dbt_project.yml` file. By default, this variable is assumed to be `true`: +```yml +# dbt_project.yml + +... +config-version: 2 + +vars: + fivetran_log: + fivetran_log_using_sync_alert_messages: false # this will disable only the sync alert messages within the connector status model +``` + ### Changing the Build Schema By default this package will build the Fivetran Log staging models within a schema titled ( + `_stg_fivetran_log`) and the Fivetran Log final models within your + `_fivetran_log` in your target database. If this is not where you would like you Fivetran Log staging and final models to be written to, add the following configuration to your `dbt_project.yml` file: diff --git a/models/fivetran_log__connector_status.sql b/models/fivetran_log__connector_status.sql index 41e8b5b6..00bc451f 100644 --- a/models/fivetran_log__connector_status.sql +++ b/models/fivetran_log__connector_status.sql @@ -121,8 +121,8 @@ final as ( coalesce(schema_changes.number_of_schema_changes_last_month, 0) as number_of_schema_changes_last_month {% if var('fivetran_log_using_sync_alert_messages', true) %} - , {{ fivetran_utils.string_agg("distinct case when connector_recent_logs.event_type = 'SEVERE' then connector_recent_logs.message_data else null end", "'\\n'") }} as errors_since_last_completed_sync - , {{ fivetran_utils.string_agg("distinct case when connector_recent_logs.event_type = 'WARNING' then connector_recent_logs.message_data else null end", "'\\n'") }} as warnings_since_last_completed_sync + , {{ fivetran_utils.string_agg("case when connector_recent_logs.event_type = 'SEVERE' then connector_recent_logs.message_data else null end", "'\\n'") }} as errors_since_last_completed_sync + , {{ fivetran_utils.string_agg("case when connector_recent_logs.event_type = 'WARNING' then connector_recent_logs.message_data else null end", "'\\n'") }} as warnings_since_last_completed_sync {% endif %} from connector_recent_logs