Merge pull request #19 from dbt-labs/feature/global-connections

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.8.0 (2024-08-30)
+
+- Add `dbtcloud_global_connection` and the ability to link it to environments
+- Remove `dbtcloud_project_connection` now that connections are set at the environment level
+
+## 0.7.0
+
+- Add import for notifications on warning for jobs
+
 ## 0.6.0 (2024-06-28)
 
 - Add support for on_merge triggers for jobs and update testing

README.md

@@ -47,14 +47,15 @@ This tool can be used to load existing dbt Cloud configuration into Terraform. C
 | dbtcloud_extended_attributes(*)            | Project        | ✅                 | ✅               |                       |
 | dbtcloud_fabric_connection                 | Project        |                    |                  |                       |
 | dbtcloud_fabric_credential                 | Project        |                    |                  | 🔒                   |
+| dbtcloud_global_connection                 | Account        | ✅                 | ✅               | 🔒*                   |
 | dbtcloud_group                             | Account        | ✅                 | ✅               |                       |
 | dbtcloud_job                               | Project        | ✅                 | ✅               |                       |
 | dbtcloud_license_map                       | Account        |                    |                  |                       |
 | dbtcloud_notification                      | Account        | ✅                 | ✅               |                       |
 | dbtcloud_postgres_credential               | Project        |                    |                  | 🔒*                  |
 | dbtcloud_project                           | Project        | ✅                 | ✅               |                       |
 | dbtcloud_project_artefacts (deprecated)    | Project        | ❌                 | ❌               |                       |
-| dbtcloud_project_connection                | Project        | ✅                 | ✅               |                       |
+| dbtcloud_project_connection (deprecated)   | Project        | ❌                 | ❌               |                       |
 | dbtcloud_project_repository                | Project        | ✅                 | ✅               |                       |
 | dbtcloud_repository                        | Project        | ✅                 | ✅               |                       |
 | dbtcloud_service_token                     | Account        | ✅                 | ✅               |                      |
@@ -64,7 +65,6 @@ This tool can be used to load existing dbt Cloud configuration into Terraform. C
 
 Notes:
 
-- `dbtcloud_connection` is supported for Snowflake, Redshift, Postgres and Databricks, but not for Spark
 - `dbtcloud_extended_attributes` currently doesn't generate config for nested fields, only top level ones
 - `dbtcloud_databricks_credential` requires manually linking the `adapter_id` with the relevant `dbtcloud_connection`
 
@@ -79,6 +79,8 @@ You can then add it to your PATH and run it with `dbtcloud-terraforming` or run
 
 To update to the latest version, you can head back to the release page and download and extract the executable again.
 
+Alternatively, you can use [`eget`](https://github.com/zyedidia/eget) to more easily download the relevant executable.
+
 #### MacOS and Linux
 
 The CLI can be installed with `brew`, running `brew install dbt-labs/dbt-cli/dbtcloud-terraforming`.

dbtcloud/api.go

@@ -109,44 +109,44 @@ func (c *DbtCloudHTTPClient) Do(req *http.Request) (*http.Response, error) {
 func (c *DbtCloudHTTPClient) GetEndpoint(url string) ([]byte, error) {
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
-		log.Fatalf("Error creating a new request: %v", err)
+		return nil, fmt.Errorf("error creating a new request: %v", err)
 	}
 
 	resp, err := c.Do(req)
 	if err != nil {
-		log.Fatalf("Error fetching URL %v: %v", url, err)
+		return nil, fmt.Errorf("error fetching URL %v: %v", url, err)
 	}
 	// Ensure the response body is closed at the end.
 	defer resp.Body.Close()
 
 	// Read the response body
 	jsonPayload, err := io.ReadAll(resp.Body)
 	if err != nil {
-		log.Fatalf("Error reading body: %v", err)
+		return nil, fmt.Errorf("error reading body: %v", err)
 	}
 
 	// 400 and more are errors, either on the client side or the server side
 	if resp.StatusCode >= 400 {
-		log.WithFields(logrus.Fields{"status": resp.Status, "body": string(jsonPayload)}).Fatalf("Error fetching URL %v", url)
+		return nil, fmt.Errorf("error fetching URL %v: %v -- body: %s", url, resp.Status, string(jsonPayload))
 	}
 
-	return jsonPayload, err
+	return jsonPayload, nil
 }
 
-func (c *DbtCloudHTTPClient) GetSingleData(url string) any {
+func (c *DbtCloudHTTPClient) GetSingleData(url string) (any, error) {
 
 	jsonPayload, err := c.GetEndpoint(url)
 	if err != nil {
-		log.Fatal(err)
+		return nil, err
 	}
 	var response SingleResponse
 
 	err = json.Unmarshal(jsonPayload, &response)
 	if err != nil {
-		log.Fatal(err)
+		return nil, err
 	}
 
-	return response.Data
+	return response.Data, nil
 }
 
 func (c *DbtCloudHTTPClient) GetData(url string) []any {
@@ -233,7 +233,7 @@ func (c *DbtCloudHTTPClient) GetProjects(listProjects []int) []any {
 		dataTyped := data.(map[string]any)
 		projectID := dataTyped["id"].(float64)
 
-		if len(listProjects) > 0 && lo.Contains(listProjects, int(projectID)) == false {
+		if len(listProjects) > 0 && !lo.Contains(listProjects, int(projectID)) {
 			continue
 		}
 		filteredProjects = append(filteredProjects, data)
@@ -262,7 +262,7 @@ func filterByProject(allData []any, listProjects []int) []any {
 		dataTyped := data.(map[string]any)
 		projectID := dataTyped["project_id"].(float64)
 
-		if lo.Contains(listProjects, int(projectID)) == false {
+		if !lo.Contains(listProjects, int(projectID)) {
 			continue
 		}
 		filteredData = append(filteredData, data)
@@ -322,7 +322,7 @@ func (c *DbtCloudHTTPClient) GetConnections(listProjects []int, warehouses []str
 		projectTyped := project.(map[string]any)
 		projectID := int(projectTyped["id"].(float64))
 
-		if len(listProjects) > 0 && lo.Contains(listProjects, projectID) == false {
+		if len(listProjects) > 0 && !lo.Contains(listProjects, projectID) {
 			continue
 		}
 
@@ -346,7 +346,12 @@ func (c *DbtCloudHTTPClient) GetConnections(listProjects []int, warehouses []str
 		}
 
 		url := fmt.Sprintf("%s/v3/accounts/%s/projects/%d/connections/%0.f/", c.HostURL, c.AccountID, projectID, projectConnectionTyped["id"].(float64))
-		connection := c.GetSingleData(url)
+		connection, err := c.GetSingleData(url)
+		if err != nil {
+			log.Warn(err)
+			continue
+		}
+
 		connections = append(connections, connection)
 	}
 
@@ -447,7 +452,7 @@ func (c *DbtCloudHTTPClient) GetExtendedAttributes(listProjects []int) []any {
 		}
 		projectID := envTyped["project_id"].(float64)
 		url := fmt.Sprintf("%s/v3/accounts/%s/projects/%0.f/extended-attributes/%0.f/", c.HostURL, c.AccountID, projectID, extendedAttributesID)
-		extendedAttributes := c.GetSingleData(url)
+		extendedAttributes, _ := c.GetSingleData(url)
 		allExtendedAttributes = append(allExtendedAttributes, extendedAttributes)
 	}
 	return allExtendedAttributes
@@ -489,3 +494,32 @@ func (c *DbtCloudHTTPClient) GetServiceTokenPermissions(serviceTokenID int) []an
 
 	return c.GetData(url)
 }
+
+func (c *DbtCloudHTTPClient) GetGlobalConnection(id int64) (any, error) {
+	url := fmt.Sprintf("%s/v3/accounts/%s/connections/%d/", c.HostURL, c.AccountID, id)
+
+	return c.GetSingleData(url)
+}
+
+func (c *DbtCloudHTTPClient) GetGlobalConnectionsSummary() []any {
+	url := fmt.Sprintf("%s/v3/accounts/%s/connections/", c.HostURL, c.AccountID)
+
+	return c.GetData(url)
+}
+
+func (c *DbtCloudHTTPClient) GetGlobalConnections() []any {
+
+	// this return just a summary though...
+	// so we need to loop through the results to get the details
+	allConnectionsSummary := c.GetGlobalConnectionsSummary()
+	allConnectionDetails := []any{}
+
+	for _, connectionSummary := range allConnectionsSummary {
+		connectionSummaryTyped := connectionSummary.(map[string]any)
+		connectionID := int(connectionSummaryTyped["id"].(float64))
+		connectionDetails, _ := c.GetGlobalConnection(int64(connectionID))
+		allConnectionDetails = append(allConnectionDetails, connectionDetails)
+	}
+
+	return allConnectionDetails
+}

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.8.4
-	github.com/zclconf/go-cty v1.13.2
+	github.com/zclconf/go-cty v1.15.0
 	golang.org/x/time v0.3.0
 	gopkg.in/dnaeon/go-vcr.v3 v3.1.2
 )
@@ -24,6 +24,7 @@ require (
 	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
 	github.com/agext/levenshtein v1.2.1 // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
+	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect

go.sum

@@ -50,6 +50,8 @@ github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tj
 github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
 github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
 github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
+github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
+github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -243,8 +245,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/zclconf/go-cty v1.13.2 h1:4GvrUxe/QUDYuJKAav4EYqdM47/kZa672LwmXFmEKT0=
-github.com/zclconf/go-cty v1.13.2/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0=
+github.com/zclconf/go-cty v1.15.0 h1:tTCRWxsexYUmtt/wVxgDClUe+uQusuI443uL6e+5sXQ=
+github.com/zclconf/go-cty v1.15.0/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

internal/app/dbtcloud-terraforming/cmd/generate.go

@@ -45,6 +45,10 @@ func generateResources() func(cmd *cobra.Command, args []string) {
 			log.Fatal("you must define at least one --resource-types to generate the config")
 		}
 
+		if len(resourceTypes) == 1 && resourceTypes[0] == "all" {
+			resourceTypes = lo.Keys(resourceImportStringFormats)
+		}
+
 		listFilterProjects = viper.GetIntSlice("projects")
 
 		var execPath, workingDir string
@@ -111,23 +115,6 @@ func generateResources() func(cmd *cobra.Command, args []string) {
 				jsonStructData = dbtCloudClient.GetProjects(listFilterProjects)
 				resourceCount = len(jsonStructData)
 
-			case "dbtcloud_project_connection":
-
-				listProjects := dbtCloudClient.GetProjects(listFilterProjects)
-
-				for _, project := range listProjects {
-					projectTyped := project.(map[string]any)
-					projectTyped["project_id"] = projectTyped["id"].(float64)
-					jsonStructData = append(jsonStructData, projectTyped)
-
-					if linkResource("dbtcloud_project") {
-						projectID := projectTyped["project_id"]
-						projectTyped["project_id"] = fmt.Sprintf("dbtcloud_project.terraform_managed_resource_%0.f.id", projectID)
-					}
-				}
-
-				resourceCount = len(jsonStructData)
-
 			case "dbtcloud_job":
 
 				jobs := dbtCloudClient.GetJobs(listFilterProjects)
@@ -230,18 +217,31 @@ func generateResources() func(cmd *cobra.Command, args []string) {
 					if credentialID, ok := environmentsTyped["credentials_id"].(float64); ok {
 
 						environmentsTyped["credential_id"] = credentialID
-						if linkResource("dbtcloud_snowflake_credential") || linkResource("dbtcloud_bigquery_credential") {
+						if linkResource("dbtcloud_snowflake_credential") || linkResource("dbtcloud_bigquery_credential") || linkResource("dbtcloud_databricks_credential") {
 
-							credentials := environmentsTyped["credentials"].(map[string]any)
-							credentialsType := credentials["type"].(string)
+							credentials, credentialsOK := environmentsTyped["credentials"].(map[string]any)
 
-							if !lo.Contains([]string{"snowflake", "bigquery"}, credentialsType) {
-								panic("Only Snowflake and BigQuery credentials are supported for now. Please raise an issue in the repo if you would like to see other adapter supported")
-							}
+							if credentialsOK {
+								credentialsType := credentials["type"].(string)
+								adapterVersion := credentials["adapter_version"].(string)
+
+								if lo.Contains([]string{"snowflake", "bigquery"}, credentialsType) {
+									environmentsTyped["credential_id"] = fmt.Sprintf("dbtcloud_%s_credential.terraform_managed_resource_%0.f.credential_id", credentialsType, credentialID)
+								} else if adapterVersion != "databricks_v0" {
+									environmentsTyped["credential_id"] = fmt.Sprintf("dbtcloud_databricks_credential.terraform_managed_resource_%0.f.credential_id", credentialID)
+								} else {
+									environmentsTyped["credential_id"] = "---TBD---credential type not supported yet---"
+								}
 
-							environmentsTyped["credential_id"] = fmt.Sprintf("dbtcloud_%s_credential.terraform_managed_resource_%0.f.credential_id", credentialsType, credentialID)
+							} else {
+								environmentsTyped["credential_id"] = "---TBD---"
+							}
 						}
 					}
+					if linkResource("dbtcloud_global_connection") {
+						connectionID := environmentsTyped["connection_id"].(float64)
+						environmentsTyped["connection_id"] = fmt.Sprintf("dbtcloud_global_connection.terraform_managed_resource_%0.f.id", connectionID)
+					}
 
 					// handle the case when extended_attributes_id is not set
 					if extendedAttributesID, ok := environmentsTyped["extended_attributes_id"].(float64); ok {
@@ -372,7 +372,7 @@ func generateResources() func(cmd *cobra.Command, args []string) {
 					case "password":
 						credentialTyped["password"] = "---TBD---"
 					case "keypair":
-						credentialTyped["private_key"] = "!!!TBD!!!"
+						credentialTyped["private_key"] = "---TBD---"
 						credentialTyped["private_key_passphrase"] = "---TBD---"
 					}
 
@@ -696,6 +696,53 @@ func generateResources() func(cmd *cobra.Command, args []string) {
 				}
 				resourceCount = len(jsonStructData)
 
+			case "dbtcloud_global_connection":
+
+				listConnections := dbtCloudClient.GetGlobalConnections()
+
+				for _, connection := range listConnections {
+					connectionTyped := connection.(map[string]any)
+					jsonStructData = append(jsonStructData, connectionTyped)
+
+					configSection := getAdapterFromAdapterVersion(connectionTyped["adapter_version"].(string))
+
+					configTyped := connectionTyped["config"].(map[string]any)
+					delete(configTyped, "adapter_id")
+
+					// handle the fields that don't come back from the API
+					_, exists := configTyped["oauth_client_id"]
+					if exists {
+						configTyped["oauth_client_id"] = "---TBD---"
+					}
+					_, exists = configTyped["oauth_client_secret"]
+					if exists {
+						configTyped["oauth_client_secret"] = "---TBD---"
+					}
+					_, exists = configTyped["private_key"]
+					if exists {
+						configTyped["private_key"] = "---TBD---"
+					}
+					_, exists = configTyped["application_id"]
+					if exists {
+						configTyped["application_id"] = "---TBD---"
+					}
+					_, exists = configTyped["application_secret"]
+					if exists {
+						configTyped["application_secret"] = "---TBD---"
+					}
+					// For BQ, to handle the renaming of the fields
+					gcpProjectID, exists := configTyped["project_id"]
+					if exists && configSection == "bigquery" {
+						configTyped["gcp_project_id"] = gcpProjectID
+						delete(configTyped, "project_id")
+					}
+
+					connectionTyped[configSection] = connectionTyped["config"]
+
+				}
+
+				resourceCount = len(jsonStructData)
+
 			default:
 				fmt.Fprintf(cmd.OutOrStderr(), "%q is not yet supported for automatic generation", resourceType)
 				return
@@ -755,6 +802,12 @@ func generateResources() func(cmd *cobra.Command, args []string) {
 						continue
 					}
 
+					// This is to handle Attributes in the Framework
+					if r.Block.Attributes[attrName].AttributeType == cty.NilType {
+						writeAttrLine(attrName, structData[attrName], "", resource)
+						continue
+					}
+
 					ty := r.Block.Attributes[attrName].AttributeType
 					switch {
 					case ty.IsPrimitiveType():