diff --git a/.changes/unreleased/Features-20221221-132810.yaml b/.changes/unreleased/Features-20221221-132810.yaml new file mode 100644 index 0000000..9e5d36b --- /dev/null +++ b/.changes/unreleased/Features-20221221-132810.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Add dbt_queries model to easily understand dbt model costs with links to dbt + Cloud +time: 2022-12-21T13:28:10.065082Z +custom: + Author: NiallRees + PR: "47" diff --git a/README.md b/README.md index 947a256..812bd42 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,12 @@ query-comment: append: true # Snowflake removes prefixed comments. ``` +The generate URLs to dbt Cloud jobs and runs in the `dbt_queries` model, add the following variable to `dbt_project.yml`: +```yaml +vars: + dbt_cloud_account_id: 12345 # https://cloud.getdbt.com/next/deploy// +``` + ## Example Usage ### Sample Queries diff --git a/documentation/sample_queries.md b/documentation/sample_queries.md index 7f37343..736c50e 100644 --- a/documentation/sample_queries.md +++ b/documentation/sample_queries.md @@ -113,18 +113,17 @@ limit 10 with max_date as ( select max(date(end_time)) as date - from query_history_enriched + from dbt_queries ) select - dbt_metadata['node_id']::string as dbt_node_id, - sum(query_history_enriched.query_cost) as total_cost_last_30d, + dbt_queries.dbt_node_id, + sum(dbt_queries.query_cost) as total_cost_last_30d, total_cost_last_30d*12 as estimated_annual_cost -from query_history_enriched +from dbt_queries cross join max_date where - query_history_enriched.start_time >= dateadd('day', -30, max_date.date) - and query_history_enriched.start_time < max_date.date -- don't include partial day of data - and dbt_metadata is not null + dbt_queries.start_time >= dateadd('day', -30, max_date.date) + and dbt_queries.start_time < max_date.date -- don't include partial day of data group by 1 order by total_cost_last_30d desc limit 10 @@ -136,8 +135,8 @@ limit 10 select date(start_time) as date, sum(query_cost) as cost -from query_history_enriched -where dbt_metadata['node_id']::string='' +from dbt_queries +where dbt_node_id = '' group by 1 order by 1 desc ``` diff --git a/models/dbt_queries.sql b/models/dbt_queries.sql new file mode 100644 index 0000000..9b2dd8f --- /dev/null +++ b/models/dbt_queries.sql @@ -0,0 +1,25 @@ +select + dbt_metadata['dbt_version']::string as dbt_version, + dbt_metadata['target_name']::string as dbt_target_name, + dbt_metadata['target_database']::string as dbt_target_database, + dbt_metadata['target_schema']::string as dbt_target_schema, + dbt_metadata['invocation_id']::string as dbt_invocation_id, + dbt_metadata['node_id']::string as dbt_node_id, + dbt_metadata['node_resource_type']::string as dbt_node_resource_type, + dbt_metadata['materialized']::string as dbt_node_materialized, + dbt_metadata['is_incremental']::string as dbt_node_is_incremental, + dbt_metadata['dbt_cloud_project_id']::string as dbt_cloud_project_id, + dbt_metadata['dbt_cloud_job_id']::string as dbt_cloud_job_id, + dbt_metadata['dbt_cloud_run_id']::string as dbt_cloud_run_id, + dbt_metadata['dbt_cloud_run_reason_category']::string as dbt_cloud_run_reason_category, + dbt_metadata['dbt_cloud_run_reason']::string as dbt_cloud_run_reason, + {% if var('dbt_cloud_account_id', none) -%} + 'https://cloud.getdbt.com/next/deploy/' || '{{ var('dbt_cloud_account_id') }}' || '/projects/' || dbt_cloud_project_id || '/jobs/' || dbt_cloud_job_id as dbt_cloud_job_url, + 'https://cloud.getdbt.com/next/deploy/' || '{{ var('dbt_cloud_account_id') }}' || '/projects/' || dbt_cloud_project_id || '/runs/' || dbt_cloud_run_id as dbt_cloud_run_url, + {%- else -%} + 'Required dbt_cloud_account_id variable not set' as dbt_cloud_job_url, + 'Required dbt_cloud_account_id variable not set' as dbt_cloud_run_url, + {%- endif %} + * exclude dbt_metadata +from {{ ref('query_history_enriched') }} +where dbt_metadata is not null diff --git a/models/dbt_queries.yml b/models/dbt_queries.yml new file mode 100644 index 0000000..87a84ad --- /dev/null +++ b/models/dbt_queries.yml @@ -0,0 +1,38 @@ +version: 2 + +models: + - name: dbt_queries + description: Filtered version of query_history_enriched just for queries issued by dbt. Adds additional dbt-specific columns. + columns: + - name: dbt_version + description: Version of dbt in use. + - name: dbt_target_name + description: The target name for the dbt invocation. + - name: dbt_target_database + description: The target database for the dbt invocation. + - name: dbt_target_schema + description: The target schema for the dbt invocation. + - name: dbt_invocation_id + description: The id of the dbt invocation. + - name: dbt_node_id + description: The identifier for the node that the query relates to. + - name: dbt_node_resource_type + description: The resource type of the node that the query relates to. + - name: dbt_node_materialized + description: The materialization of the node that the query relates to. + - name: dbt_node_is_incremental + description: The materialization of the node that the query relates to. + - name: dbt_cloud_project_id + description: If using dbt Cloud, the ID of the project. + - name: dbt_cloud_job_id + description: If using dbt Cloud, the ID of the issuing job. + - name: dbt_cloud_run_id + description: If using dbt Cloud, the ID of the issuing run. + - name: dbt_cloud_run_reason_category + description: If using dbt Cloud, the run reason category for the issuing run. + - name: dbt_cloud_run_reason + description: If using dbt Cloud, the run reason for the issuing run. + - name: dbt_cloud_job_url + description: If using dbt Cloud, the URL of the issuing job. The dbt_cloud_account_id dbt variable must be set for this field to populate. + - name: dbt_cloud_run_url + description: If using dbt Cloud, the URL of the issuing run. The dbt_cloud_account_id dbt variable must be set for this field to populate. diff --git a/tox.ini b/tox.ini index 0bb0dc1..0a02758 100644 --- a/tox.ini +++ b/tox.ini @@ -105,9 +105,9 @@ changedir = integration_test_project deps = dbt-snowflake~=1.2.0 commands = dbt deps - dbt build -s dbt_snowflake_monitoring --full-refresh + dbt build -s +daily_spend --full-refresh dbt build --exclude dbt_snowflake_monitoring --full-refresh - dbt build -s dbt_snowflake_monitoring + dbt build -s +daily_spend dbt build --exclude dbt_snowflake_monitoring [testenv:snowflake]