diff --git a/CHANGELOG.md b/CHANGELOG.md index 862978f..0e67f95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +# Changelog + +## 0.4.0 (2024-01-23) + +- Add support for `dbtcloud_webhook` and `dbtcloud_notification` + +## 0.3.0 + +- Add support for `dbtcloud_user_groups` + +## 0.2.0 + +- Add CI/CD to release to homebrew + ## 0.1.0 (2023-11-30) - Initial creation of the repo, a modified copy of `cf-terraforming` diff --git a/README.md b/README.md index 38b5fda..ed00f5f 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ This tool can be used to load existing dbt Cloud configuration into Terraform. C | dbtcloud_connection | Project | ✅ | ✅ | 🔒* | | dbtcloud_databricks_credential | Project | | | | | dbtcloud_environment | Project | ✅ | ✅ | | -| dbtcloud_environment_variable | Project | ✅ | ✅ | 🔒* | +| dbtcloud_environment_variable | Project | ✅ | ✅ | 🔒* | | dbtcloud_environment_variable_job_override | Project | | | | | dbtcloud_extended_attributes(*) | Project | ✅ | ✅ | | | dbtcloud_fabric_connection | Project | | | | @@ -50,7 +50,7 @@ This tool can be used to load existing dbt Cloud configuration into Terraform. C | dbtcloud_group | Account | ✅ | ✅ | | | dbtcloud_job | Project | ✅ | ✅ | | | dbtcloud_license_map | Account | | | | -| dbtcloud_notification | Account | | | | +| dbtcloud_notification | Account | ✅ | ✅ | | | dbtcloud_postgres_credential | Project | | | 🔒* | | dbtcloud_project | Project | ✅ | ✅ | | | dbtcloud_project_artefacts | Project | | | | @@ -60,7 +60,7 @@ This tool can be used to load existing dbt Cloud configuration into Terraform. C | dbtcloud_service_token | Account | | | 🔒 | | dbtcloud_snowflake_credential | Project | ✅ | ✅ | 🔒 | | dbtcloud_user_groups | Account | ✅ | ✅ | | -| dbtcloud_webhook | Account | | | | +| dbtcloud_webhook | Account | ✅ | ✅ | | Notes: diff --git a/dbtcloud/api.go b/dbtcloud/api.go index e179238..1051d07 100644 --- a/dbtcloud/api.go +++ b/dbtcloud/api.go @@ -4,12 +4,12 @@ import ( "encoding/json" "fmt" "io" - "log" "net/http" "strings" "time" "github.com/samber/lo" + "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) @@ -58,6 +58,8 @@ type RateLimitedTransport struct { limiter *rate.Limiter } +var log = logrus.New() + // RoundTrip overrides the http.RoundTrip to implement rate limiting. func (t *RateLimitedTransport) RoundTrip(req *http.Request) (*http.Response, error) { // Wait for permission from the rate limiter @@ -117,7 +119,7 @@ func (c *DbtCloudHTTPClient) GetEndpoint(url string) (error, []byte) { // 400 and more are errors, either on the client side or the server side if resp.StatusCode >= 400 { - log.Fatalf("Error fetching URL %v: Status %v -- %v", url, resp.Status, string(jsonPayload)) + log.WithFields(logrus.Fields{"status": resp.Status, "body": string(jsonPayload)}).Fatalf("Error fetching URL %v", url) } return err, jsonPayload @@ -422,3 +424,15 @@ func (c *DbtCloudHTTPClient) GetUsers() []any { return c.GetData(url) } + +func (c *DbtCloudHTTPClient) GetWebhooks() []any { + url := fmt.Sprintf("%s/v3/accounts/%s/webhooks/subscriptions", c.HostURL, c.AccountID) + + return c.GetData(url) +} + +func (c *DbtCloudHTTPClient) GetNotifications() []any { + url := fmt.Sprintf("%s/v2/accounts/%s/notifications/", c.HostURL, c.AccountID) + + return c.GetData(url) +} diff --git a/internal/app/dbtcloud-terraforming/cmd/generate.go b/internal/app/dbtcloud-terraforming/cmd/generate.go index eee8cec..b69a7f6 100644 --- a/internal/app/dbtcloud-terraforming/cmd/generate.go +++ b/internal/app/dbtcloud-terraforming/cmd/generate.go @@ -545,10 +545,62 @@ func generateResources() func(cmd *cobra.Command, args []string) { jsonStructData = append(jsonStructData, userTyped) } - // jsonStructData = listUsers resourceCount = len(jsonStructData) + case "dbtcloud_webhook": + + listWebhooks := dbtCloudClient.GetWebhooks() + for _, webhook := range listWebhooks { + webhookTyped := webhook.(map[string]any) + + if linkResource("dbtcloud_job") { + jobIDs := []string{} + for _, jobID := range webhookTyped["job_ids"].([]any) { + jobIDTyped := jobID.(string) + jobIDs = append(jobIDs, jobIDTyped) + } + linkedJobIDs := lo.Map(jobIDs, func(s string, index int) string { + return fmt.Sprintf("dbtcloud_job.terraform_managed_resource_%s.id", s) + }) + webhookTyped["job_ids"] = linkedJobIDs + } + + jsonStructData = append(jsonStructData, webhookTyped) + } + resourceCount = len(jsonStructData) + + case "dbtcloud_notification": + + listNotifications := dbtCloudClient.GetNotifications() + for _, notification := range listNotifications { + notificationTyped := notification.(map[string]any) + + notificationTyped["notification_type"] = notificationTyped["type"].(float64) + notificationTyped["state"] = nil + + if linkResource("dbtcloud_job") { + listOns := []string{"on_cancel", "on_failure", "on_success"} + + for _, notifHook := range listOns { + + jobIDs := []float64{} + + for _, jobID := range notificationTyped[notifHook].([]any) { + jobIDTyped := jobID.(float64) + jobIDs = append(jobIDs, jobIDTyped) + } + linkedJobIDs := lo.Map(jobIDs, func(f float64, index int) string { + return fmt.Sprintf("dbtcloud_job.terraform_managed_resource_%.0f.id", f) + }) + notificationTyped[notifHook] = linkedJobIDs + } + } + + jsonStructData = append(jsonStructData, notificationTyped) + } + resourceCount = len(jsonStructData) + default: fmt.Fprintf(cmd.OutOrStderr(), "%q is not yet supported for automatic generation", resourceType) return diff --git a/internal/app/dbtcloud-terraforming/cmd/import.go b/internal/app/dbtcloud-terraforming/cmd/import.go index b05b656..a42a897 100644 --- a/internal/app/dbtcloud-terraforming/cmd/import.go +++ b/internal/app/dbtcloud-terraforming/cmd/import.go @@ -28,6 +28,8 @@ var resourceImportStringFormats = map[string]string{ "dbtcloud_connection": ":project_id::id", "dbtcloud_extended_attributes": ":project_id::id", "dbtcloud_user_groups": ":user_id", + "dbtcloud_webhook": ":id", + "dbtcloud_notification": ":id", } func init() { @@ -131,6 +133,12 @@ func runImport() func(cmd *cobra.Command, args []string) { case "dbtcloud_user_groups": jsonStructData = dbtCloudClient.GetUsers() + case "dbtcloud_webhook": + jsonStructData = dbtCloudClient.GetWebhooks() + + case "dbtcloud_notification": + jsonStructData = dbtCloudClient.GetNotifications() + default: fmt.Fprintf(cmd.OutOrStderr(), "%q is not yet supported for state import", resourceType) return diff --git a/internal/app/dbtcloud-terraforming/cmd/import_test.go b/internal/app/dbtcloud-terraforming/cmd/import_test.go index a3c5a18..dadef5a 100644 --- a/internal/app/dbtcloud-terraforming/cmd/import_test.go +++ b/internal/app/dbtcloud-terraforming/cmd/import_test.go @@ -23,8 +23,10 @@ func TestResourceImport(t *testing.T) { projects string }{ // account level resource - "dbt Cloud groups": {resourceTypes: "dbtcloud_group", testdataFilename: "dbtcloud_group"}, - "dbt Cloud user groups": {resourceTypes: "dbtcloud_user_groups", testdataFilename: "dbtcloud_user_groups"}, + "dbt Cloud groups": {resourceTypes: "dbtcloud_group", testdataFilename: "dbtcloud_group"}, + "dbt Cloud user groups": {resourceTypes: "dbtcloud_user_groups", testdataFilename: "dbtcloud_user_groups"}, + "dbt Cloud webhooks": {resourceTypes: "dbtcloud_webhook", testdataFilename: "dbtcloud_webhook"}, + "dbt Cloud notifications": {resourceTypes: "dbtcloud_notification", testdataFilename: "dbtcloud_notification"}, // single resource "dbt Cloud BigQuery connection": {resourceTypes: "dbtcloud_bigquery_connection", testdataFilename: "dbtcloud_bigquery_connection", changesExpected: []string{"private_key", "application_id", "private_key"}}, "dbt Cloud BigQuery credentials": {resourceTypes: "dbtcloud_bigquery_credential", testdataFilename: "dbtcloud_bigquery_credential"}, @@ -44,6 +46,7 @@ func TestResourceImport(t *testing.T) { // multiple at once "dbt Cloud environments and extended attributes": {resourceTypes: "dbtcloud_environment,dbtcloud_extended_attributes", testdataFilename: "dbtcloud_env_extended_attributes", listLinkedResources: "dbtcloud_extended_attributes", projects: "2570"}, "dbt Cloud projects and envs": {resourceTypes: "dbtcloud_project,dbtcloud_environment", testdataFilename: "dbtcloud_project_env", listLinkedResources: "dbtcloud_project"}, + "dbt Cloud webhooks and jobs": {resourceTypes: "dbtcloud_webhook,dbtcloud_job", testdataFilename: "dbtcloud_webhook_job", listLinkedResources: "dbtcloud_job"}, } for name, tc := range tests {