Skip to content

data-platform-hq/terraform-databricks-external-location

Repository files navigation

Databricks External Location Terraform module

Terraform module for creation Databricks External Location

Usage

# Prerequisite resources

# Databricks Workspace with Premium SKU
data "azurerm_databricks_workspace" "example" {
  name                = "example-workspace"
  resource_group_name = "example-rg"
}

resource "azurerm_databricks_access_connector" "example" {
  name                = "example-resource"
  resource_group_name = azurerm_resource_group.example.name
  location            = azurerm_resource_group.example.location

  identity {
    type = "SystemAssigned"
  }
}

# Databricks Provider configuration
provider "databricks" {
  alias                       = "main"
  host                        = data.azurerm_databricks_workspace.example.workspace_url
  azure_workspace_resource_id = data.azurerm_databricks_workspace.example.id
}

locals {
  storage_credentials = {
    prefix                    = "example"
    azure_access_connector_id = azurerm_databricks_access_connector.example.id
    permissions               = [{ principal = "ALL_PRIVILEGES_GROUP", privileges = ["ALL_PRIVILEGES"] }]
  }
  
  external_locations = 
    {
      name        = "adls-example"
      url         = "abfss://container@storageaccount.dfs.core.windows.net"
      permissions = [
        { principal = "ALL_PRIVILEGES_GROUP", privileges = ["ALL_PRIVILEGES"] },
        { principaprincipal = "EXAMPLE_PERMISSION_GROUP", privileges = ["CREATE_EXTERNAL_TABLE", "READ_FILES"] }
      ]    
      owner           = "username@domain.com"
      skip_validation = true
      read_only       = false
      comment         = "example_comment"
    }    
}

# Databricks External Location 
module "databricks_locations" {
  count  = var.databricks_configure ? (module.databricks_workspace.sku == "premium" ? 1 : 0) : 0

  source  = "data-platform-hq/external-location/databricks"
  version  = "~> 1.0"

  project            = "datahq"
  env                = "example"
  location           = "eastus"
  storage_credential = local.storage_credentials
  external_locations = local.external_locations

  providers = {
    databricks = databricks.workspace
  }
}

Requirements

Name Version
terraform >=1.0.0
azurerm >= 4.0.1
databricks ~>1.0

Providers

Name Version
databricks ~>1.0

Modules

No modules.

Resources

Name Type
databricks_external_location.this resource
databricks_grants.credential resource
databricks_grants.locations resource
databricks_storage_credential.this resource

Inputs

Name Description Type Default Required
external_locations List of object with external location configuration attributes
list(object({
index = string # Index of instance, for example short name, used later to access exact external location in output map
name = string # Custom whole name of resource
url = string # Path URL in cloud storage
credentials_name = optional(string)
owner = optional(string) # Owner of resource
skip_validation = optional(bool, true) # Suppress validation errors if any & force save the external location
read_only = optional(bool, false) # Indicates whether the external location is read-only.
force_destroy = optional(bool, true)
force_update = optional(bool, true)
comment = optional(string, "External location provisioned by Terraform")
permissions = optional(set(object({
principal = string
privileges = list(string)
})), [])
isolation_mode = optional(string, "ISOLATION_MODE_OPEN")
}))
[] no
storage_credential Object with storage credentials configuration attributes
object({
azure_access_connector_id = optional(string, null) # Azure Databricks Access Connector Id
cloud = optional(string, "")
name = optional(string, null) # Custom whole name of resource
owner = optional(string) # Owner of resource
force_destroy = optional(bool, true)
comment = optional(string, "Managed identity credential provisioned by Terraform")
permissions = optional(set(object({
principal = string
privileges = list(string)
})), [])
isolation_mode = optional(string, "ISOLATION_MODE_OPEN")
})
n/a yes

Outputs

Name Description
databricks_gcp_service_account The email of the GCP service account created, to be granted access to relevant buckets
external_locations Map of objects with External Location parameters, like name, credentials name and url of target storage
storage_credential_metastore_id Storage Credential metastore id
storage_credential_name Storage Credential name

License

Apache 2 Licensed. For more information please see LICENSE