Merge pull request #11 from dbt-labs/feature/add-webhooks-notifications

Add webhooks and notifications

CHANGELOG.md

@@ -1,3 +1,17 @@
+# Changelog
+
+## 0.4.0 (2024-01-23)
+
+- Add support for `dbtcloud_webhook` and `dbtcloud_notification`
+
+## 0.3.0
+
+- Add support for `dbtcloud_user_groups`
+
+## 0.2.0
+
+- Add CI/CD to release to homebrew
+
 ## 0.1.0 (2023-11-30)
 
 - Initial creation of the repo, a modified copy of `cf-terraforming`

README.md

@@ -42,15 +42,15 @@ This tool can be used to load existing dbt Cloud configuration into Terraform. C
 | dbtcloud_connection                        | Project        | ✅                 | ✅               | 🔒*                   |
 | dbtcloud_databricks_credential             | Project        |                    |                  |                       |
 | dbtcloud_environment                       | Project        | ✅                 | ✅               |                       |
-| dbtcloud_environment_variable              | Project        | ✅                 | ✅               | 🔒*                    |
+| dbtcloud_environment_variable              | Project        | ✅                 | ✅               | 🔒*                   |
 | dbtcloud_environment_variable_job_override | Project        |                    |                  |                       |
 | dbtcloud_extended_attributes(*)            | Project        | ✅                 | ✅               |                       |
 | dbtcloud_fabric_connection                 | Project        |                    |                  |                       |
 | dbtcloud_fabric_credential                 | Project        |                    |                  | 🔒                    |
 | dbtcloud_group                             | Account        | ✅                 | ✅               |                       |
 | dbtcloud_job                               | Project        | ✅                 | ✅               |                       |
 | dbtcloud_license_map                       | Account        |                    |                  |                       |
-| dbtcloud_notification                      | Account        |                    |                  |                       |
+| dbtcloud_notification                      | Account        | ✅                 | ✅               |                       |
 | dbtcloud_postgres_credential               | Project        |                    |                  | 🔒*                   |
 | dbtcloud_project                           | Project        | ✅                 | ✅               |                       |
 | dbtcloud_project_artefacts                 | Project        |                    |                  |                       |
@@ -60,7 +60,7 @@ This tool can be used to load existing dbt Cloud configuration into Terraform. C
 | dbtcloud_service_token                     | Account        |                    |                  | 🔒                    |
 | dbtcloud_snowflake_credential              | Project        | ✅                 | ✅               | 🔒                    |
 | dbtcloud_user_groups                       | Account        | ✅                 | ✅               |                       |
-| dbtcloud_webhook                           | Account        |                    |                  |                       |
+| dbtcloud_webhook                           | Account        | ✅                 | ✅               |                       |
 
 Notes:
 

dbtcloud/api.go

@@ -4,12 +4,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"strings"
 	"time"
 
 	"github.com/samber/lo"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/time/rate"
 )
 
@@ -58,6 +58,8 @@ type RateLimitedTransport struct {
 	limiter *rate.Limiter
 }
 
+var log = logrus.New()
+
 // RoundTrip overrides the http.RoundTrip to implement rate limiting.
 func (t *RateLimitedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	// Wait for permission from the rate limiter
@@ -117,7 +119,7 @@ func (c *DbtCloudHTTPClient) GetEndpoint(url string) (error, []byte) {
 
 	// 400 and more are errors, either on the client side or the server side
 	if resp.StatusCode >= 400 {
-		log.Fatalf("Error fetching URL %v: Status %v -- %v", url, resp.Status, string(jsonPayload))
+		log.WithFields(logrus.Fields{"status": resp.Status, "body": string(jsonPayload)}).Fatalf("Error fetching URL %v", url)
 	}
 
 	return err, jsonPayload
@@ -422,3 +424,15 @@ func (c *DbtCloudHTTPClient) GetUsers() []any {
 
 	return c.GetData(url)
 }
+
+func (c *DbtCloudHTTPClient) GetWebhooks() []any {
+	url := fmt.Sprintf("%s/v3/accounts/%s/webhooks/subscriptions", c.HostURL, c.AccountID)
+
+	return c.GetData(url)
+}
+
+func (c *DbtCloudHTTPClient) GetNotifications() []any {
+	url := fmt.Sprintf("%s/v2/accounts/%s/notifications/", c.HostURL, c.AccountID)
+
+	return c.GetData(url)
+}

internal/app/dbtcloud-terraforming/cmd/generate.go

@@ -545,10 +545,62 @@ func generateResources() func(cmd *cobra.Command, args []string) {
 
 					jsonStructData = append(jsonStructData, userTyped)
 				}
-				// jsonStructData = listUsers
 
 				resourceCount = len(jsonStructData)
 
+			case "dbtcloud_webhook":
+
+				listWebhooks := dbtCloudClient.GetWebhooks()
+				for _, webhook := range listWebhooks {
+					webhookTyped := webhook.(map[string]any)
+
+					if linkResource("dbtcloud_job") {
+						jobIDs := []string{}
+						for _, jobID := range webhookTyped["job_ids"].([]any) {
+							jobIDTyped := jobID.(string)
+							jobIDs = append(jobIDs, jobIDTyped)
+						}
+						linkedJobIDs := lo.Map(jobIDs, func(s string, index int) string {
+							return fmt.Sprintf("dbtcloud_job.terraform_managed_resource_%s.id", s)
+						})
+						webhookTyped["job_ids"] = linkedJobIDs
+					}
+
+					jsonStructData = append(jsonStructData, webhookTyped)
+				}
+				resourceCount = len(jsonStructData)
+
+			case "dbtcloud_notification":
+
+				listNotifications := dbtCloudClient.GetNotifications()
+				for _, notification := range listNotifications {
+					notificationTyped := notification.(map[string]any)
+
+					notificationTyped["notification_type"] = notificationTyped["type"].(float64)
+					notificationTyped["state"] = nil
+
+					if linkResource("dbtcloud_job") {
+						listOns := []string{"on_cancel", "on_failure", "on_success"}
+
+						for _, notifHook := range listOns {
+
+							jobIDs := []float64{}
+
+							for _, jobID := range notificationTyped[notifHook].([]any) {
+								jobIDTyped := jobID.(float64)
+								jobIDs = append(jobIDs, jobIDTyped)
+							}
+							linkedJobIDs := lo.Map(jobIDs, func(f float64, index int) string {
+								return fmt.Sprintf("dbtcloud_job.terraform_managed_resource_%.0f.id", f)
+							})
+							notificationTyped[notifHook] = linkedJobIDs
+						}
+					}
+
+					jsonStructData = append(jsonStructData, notificationTyped)
+				}
+				resourceCount = len(jsonStructData)
+
 			default:
 				fmt.Fprintf(cmd.OutOrStderr(), "%q is not yet supported for automatic generation", resourceType)
 				return

internal/app/dbtcloud-terraforming/cmd/import.go

@@ -28,6 +28,8 @@ var resourceImportStringFormats = map[string]string{
 	"dbtcloud_connection":           ":project_id::id",
 	"dbtcloud_extended_attributes":  ":project_id::id",
 	"dbtcloud_user_groups":          ":user_id",
+	"dbtcloud_webhook":              ":id",
+	"dbtcloud_notification":         ":id",
 }
 
 func init() {
@@ -131,6 +133,12 @@ func runImport() func(cmd *cobra.Command, args []string) {
 			case "dbtcloud_user_groups":
 				jsonStructData = dbtCloudClient.GetUsers()
 
+			case "dbtcloud_webhook":
+				jsonStructData = dbtCloudClient.GetWebhooks()
+
+			case "dbtcloud_notification":
+				jsonStructData = dbtCloudClient.GetNotifications()
+
 			default:
 				fmt.Fprintf(cmd.OutOrStderr(), "%q is not yet supported for state import", resourceType)
 				return

internal/app/dbtcloud-terraforming/cmd/import_test.go

@@ -23,8 +23,10 @@ func TestResourceImport(t *testing.T) {
 		projects            string
 	}{
 		// account level resource
-		"dbt Cloud groups":      {resourceTypes: "dbtcloud_group", testdataFilename: "dbtcloud_group"},
-		"dbt Cloud user groups": {resourceTypes: "dbtcloud_user_groups", testdataFilename: "dbtcloud_user_groups"},
+		"dbt Cloud groups":        {resourceTypes: "dbtcloud_group", testdataFilename: "dbtcloud_group"},
+		"dbt Cloud user groups":   {resourceTypes: "dbtcloud_user_groups", testdataFilename: "dbtcloud_user_groups"},
+		"dbt Cloud webhooks":      {resourceTypes: "dbtcloud_webhook", testdataFilename: "dbtcloud_webhook"},
+		"dbt Cloud notifications": {resourceTypes: "dbtcloud_notification", testdataFilename: "dbtcloud_notification"},
 		// single resource
 		"dbt Cloud BigQuery connection":   {resourceTypes: "dbtcloud_bigquery_connection", testdataFilename: "dbtcloud_bigquery_connection", changesExpected: []string{"private_key", "application_id", "private_key"}},
 		"dbt Cloud BigQuery credentials":  {resourceTypes: "dbtcloud_bigquery_credential", testdataFilename: "dbtcloud_bigquery_credential"},
@@ -44,6 +46,7 @@ func TestResourceImport(t *testing.T) {
 		// multiple at once
 		"dbt Cloud environments and extended attributes": {resourceTypes: "dbtcloud_environment,dbtcloud_extended_attributes", testdataFilename: "dbtcloud_env_extended_attributes", listLinkedResources: "dbtcloud_extended_attributes", projects: "2570"},
 		"dbt Cloud projects and envs":                    {resourceTypes: "dbtcloud_project,dbtcloud_environment", testdataFilename: "dbtcloud_project_env", listLinkedResources: "dbtcloud_project"},
+		"dbt Cloud webhooks and jobs":                    {resourceTypes: "dbtcloud_webhook,dbtcloud_job", testdataFilename: "dbtcloud_webhook_job", listLinkedResources: "dbtcloud_job"},
 	}
 
 	for name, tc := range tests {