Add dbt_queries model (#47)

* Add dbt_queries model

* Update docs

* Add changelog entry

* Fix tests

* Remove dbt_metadata column

.changes/unreleased/Features-20221221-132810.yaml

@@ -0,0 +1,7 @@
+kind: Features
+body: Add dbt_queries model to easily understand dbt model costs with links to dbt
+  Cloud
+time: 2022-12-21T13:28:10.065082Z
+custom:
+  Author: NiallRees
+  PR: "47"

README.md

@@ -26,6 +26,12 @@ query-comment:
   append: true # Snowflake removes prefixed comments.
 ```
 
+The generate URLs to dbt Cloud jobs and runs in the `dbt_queries` model, add the following variable to `dbt_project.yml`:
+```yaml
+vars:
+  dbt_cloud_account_id: 12345 # https://cloud.getdbt.com/next/deploy/<this_number>/
+```
+
 ## Example Usage
 
 ### Sample Queries

documentation/sample_queries.md

@@ -113,18 +113,17 @@ limit 10
 with
 max_date as (
     select max(date(end_time)) as date
-    from query_history_enriched
+    from dbt_queries
 )
 select
-    dbt_metadata['node_id']::string as dbt_node_id,
-    sum(query_history_enriched.query_cost) as total_cost_last_30d,
+    dbt_queries.dbt_node_id,
+    sum(dbt_queries.query_cost) as total_cost_last_30d,
     total_cost_last_30d*12 as estimated_annual_cost
-from query_history_enriched
+from dbt_queries
 cross join max_date
 where
-    query_history_enriched.start_time >= dateadd('day', -30, max_date.date)
-    and query_history_enriched.start_time < max_date.date -- don't include partial day of data
-    and dbt_metadata is not null
+    dbt_queries.start_time >= dateadd('day', -30, max_date.date)
+    and dbt_queries.start_time < max_date.date -- don't include partial day of data
 group by 1
 order by total_cost_last_30d desc
 limit 10
@@ -136,8 +135,8 @@ limit 10
 select
     date(start_time) as date,
     sum(query_cost) as cost
-from query_history_enriched
-where dbt_metadata['node_id']::string='<dbt model node id>'
+from dbt_queries
+where dbt_node_id = '<dbt model node id>'
 group by 1
 order by 1 desc
 ```

models/dbt_queries.sql

@@ -0,0 +1,25 @@
+select
+    dbt_metadata['dbt_version']::string as dbt_version,
+    dbt_metadata['target_name']::string as dbt_target_name,
+    dbt_metadata['target_database']::string as dbt_target_database,
+    dbt_metadata['target_schema']::string as dbt_target_schema,
+    dbt_metadata['invocation_id']::string as dbt_invocation_id,
+    dbt_metadata['node_id']::string as dbt_node_id,
+    dbt_metadata['node_resource_type']::string as dbt_node_resource_type,
+    dbt_metadata['materialized']::string as dbt_node_materialized,
+    dbt_metadata['is_incremental']::string as dbt_node_is_incremental,
+    dbt_metadata['dbt_cloud_project_id']::string as dbt_cloud_project_id,
+    dbt_metadata['dbt_cloud_job_id']::string as dbt_cloud_job_id,
+    dbt_metadata['dbt_cloud_run_id']::string as dbt_cloud_run_id,
+    dbt_metadata['dbt_cloud_run_reason_category']::string as dbt_cloud_run_reason_category,
+    dbt_metadata['dbt_cloud_run_reason']::string as dbt_cloud_run_reason,
+    {% if var('dbt_cloud_account_id', none) -%}
+    'https://cloud.getdbt.com/next/deploy/' || '{{ var('dbt_cloud_account_id') }}' || '/projects/' || dbt_cloud_project_id || '/jobs/' || dbt_cloud_job_id as dbt_cloud_job_url,
+    'https://cloud.getdbt.com/next/deploy/' || '{{ var('dbt_cloud_account_id') }}' || '/projects/' || dbt_cloud_project_id || '/runs/' || dbt_cloud_run_id as dbt_cloud_run_url,
+    {%- else -%}
+    'Required dbt_cloud_account_id variable not set' as dbt_cloud_job_url,
+    'Required dbt_cloud_account_id variable not set' as dbt_cloud_run_url,
+    {%- endif %}
+    * exclude dbt_metadata
+from {{ ref('query_history_enriched') }}
+where dbt_metadata is not null

models/dbt_queries.yml

@@ -0,0 +1,38 @@
+version: 2
+
+models:
+  - name: dbt_queries
+    description: Filtered version of query_history_enriched just for queries issued by dbt. Adds additional dbt-specific columns.
+    columns:
+      - name: dbt_version
+        description: Version of dbt in use.
+      - name: dbt_target_name
+        description: The target name for the dbt invocation.
+      - name: dbt_target_database
+        description: The target database for the dbt invocation.
+      - name: dbt_target_schema
+        description: The target schema for the dbt invocation.
+      - name: dbt_invocation_id
+        description: The id of the dbt invocation.
+      - name: dbt_node_id
+        description: The identifier for the node that the query relates to.
+      - name: dbt_node_resource_type
+        description: The resource type of the node that the query relates to.
+      - name: dbt_node_materialized
+        description: The materialization of the node that the query relates to.
+      - name: dbt_node_is_incremental
+        description: The materialization of the node that the query relates to.
+      - name: dbt_cloud_project_id
+        description: If using dbt Cloud, the ID of the project.
+      - name: dbt_cloud_job_id
+        description: If using dbt Cloud, the ID of the issuing job.
+      - name: dbt_cloud_run_id
+        description: If using dbt Cloud, the ID of the issuing run.
+      - name: dbt_cloud_run_reason_category
+        description: If using dbt Cloud, the run reason category for the issuing run.
+      - name: dbt_cloud_run_reason
+        description: If using dbt Cloud, the run reason for the issuing run.
+      - name: dbt_cloud_job_url
+        description: If using dbt Cloud, the URL of the issuing job. The dbt_cloud_account_id dbt variable must be set for this field to populate.
+      - name: dbt_cloud_run_url
+        description: If using dbt Cloud, the URL of the issuing run. The dbt_cloud_account_id dbt variable must be set for this field to populate.

tox.ini

@@ -105,9 +105,9 @@ changedir = integration_test_project
 deps = dbt-snowflake~=1.2.0
 commands =
     dbt deps
-    dbt build -s dbt_snowflake_monitoring --full-refresh
+    dbt build -s +daily_spend --full-refresh
     dbt build --exclude dbt_snowflake_monitoring --full-refresh
-    dbt build -s dbt_snowflake_monitoring
+    dbt build -s +daily_spend
     dbt build --exclude dbt_snowflake_monitoring
 
 [testenv:snowflake]