draft for e.h.metastore

adb-external-hive-metastore/README.md

@@ -0,0 +1,6 @@
+# ADB workspace with external hive metastore
+
+This template creates:
+1. SQL Server
+2. SQL database
+3. ADB workspace

adb-external-hive-metastore/main.tf

@@ -0,0 +1,70 @@
+/**
+ * Azure Databricks workspace in custom VNet
+ *
+ * Module creates:
+ * * Resource group with random prefix
+ * * Tags, including `Owner`, which is taken from `az account show --query user`
+ * * VNet with public and private subnet
+ * * Databricks workspace
+ * * External Hive Metastore for ADB workspace
+ */
+provider "azurerm" {
+  features {}
+}
+
+provider "random" {
+}
+
+resource "random_string" "naming" {
+  special = false
+  upper   = false
+  length  = 6
+}
+
+data "azurerm_client_config" "current" {
+}
+
+data "external" "me" {
+  program = ["az", "account", "show", "--query", "user"]
+}
+
+locals {
+  // dltp - databricks labs terraform provider
+  prefix   = join("-", [var.workspace_prefix, "${random_string.naming.result}"])
+  location = var.rglocation
+  cidr     = var.spokecidr
+  dbfsname = join("", [var.dbfs_prefix, "${random_string.naming.result}"]) // dbfs name must not have special chars
+
+  // tags that are propagated down to all resources
+  tags = {
+    Environment = "Testing"
+    Owner       = lookup(data.external.me.result, "name")
+    Epoch       = random_string.naming.result
+  }
+}
+
+resource "azurerm_resource_group" "this" {
+  name     = "adb-dev-${local.prefix}-rg"
+  location = local.location
+  tags     = local.tags
+}
+
+output "arm_client_id" {
+  value = data.azurerm_client_config.current.client_id
+}
+
+output "arm_subscription_id" {
+  value = data.azurerm_client_config.current.subscription_id
+}
+
+output "arm_tenant_id" {
+  value = data.azurerm_client_config.current.tenant_id
+}
+
+output "azure_region" {
+  value = local.location
+}
+
+output "resource_group" {
+  value = azurerm_resource_group.this.name
+}

adb-external-hive-metastore/outputs.tf

@@ -0,0 +1,10 @@
+output "databricks_azure_workspace_resource_id" {
+  // The ID of the Databricks Workspace in the Azure management plane.
+  value = azurerm_databricks_workspace.this.id
+}
+
+output "workspace_url" {
+  // The workspace URL which is of the format 'adb-{workspaceId}.{random}.azuredatabricks.net'
+  // this is not named as DATABRICKS_HOST, because it affect authentication
+  value = "https://${azurerm_databricks_workspace.this.workspace_url}/"
+}

adb-external-hive-metastore/versions.tf

@@ -0,0 +1,14 @@
+# versions.tf
+terraform {
+  required_providers {
+    databricks = {
+      source  = "databrickslabs/databricks"
+      version = "0.3.10"
+    }
+
+    azurerm = {
+      source = "hashicorp/azurerm"
+      version = "=2.83.0"
+    }
+  }
+}

adb-external-hive-metastore/vnet.tf

@@ -0,0 +1,137 @@
+resource "azurerm_virtual_network" "this" {
+  name                = "${local.prefix}-vnet"
+  location            = azurerm_resource_group.this.location
+  resource_group_name = azurerm_resource_group.this.name
+  address_space       = [local.cidr]
+  tags                = local.tags
+}
+
+resource "azurerm_network_security_group" "this" {
+  name                = "${local.prefix}-nsg"
+  location            = azurerm_resource_group.this.location
+  resource_group_name = azurerm_resource_group.this.name
+  tags                = local.tags
+}
+
+resource "azurerm_network_security_rule" "aad" {
+  name                        = "AllowAAD"
+  priority                    = 200
+  direction                   = "Outbound"
+  access                      = "Allow"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "443"
+  source_address_prefix       = "VirtualNetwork"
+  destination_address_prefix  = "AzureActiveDirectory"
+  resource_group_name         = azurerm_resource_group.this.name
+  network_security_group_name = azurerm_network_security_group.this.name
+}
+
+resource "azurerm_network_security_rule" "azfrontdoor" {
+  name                        = "AllowAzureFrontDoor"
+  priority                    = 201
+  direction                   = "Outbound"
+  access                      = "Allow"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "443"
+  source_address_prefix       = "VirtualNetwork"
+  destination_address_prefix  = "AzureFrontDoor.Frontend"
+  resource_group_name         = azurerm_resource_group.this.name
+  network_security_group_name = azurerm_network_security_group.this.name
+}
+resource "azurerm_subnet" "public" {
+  name                 = "${local.prefix}-public"
+  resource_group_name  = azurerm_resource_group.this.name
+  virtual_network_name = azurerm_virtual_network.this.name
+  address_prefixes     = [cidrsubnet(local.cidr, 3, 0)]
+
+  delegation {
+    name = "databricks"
+    service_delegation {
+      name = "Microsoft.Databricks/workspaces"
+      actions = [
+        "Microsoft.Network/virtualNetworks/subnets/join/action",
+        "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action",
+      "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action"]
+    }
+  }
+}
+
+resource "azurerm_subnet_network_security_group_association" "public" {
+  subnet_id                 = azurerm_subnet.public.id
+  network_security_group_id = azurerm_network_security_group.this.id
+}
+
+variable "private_subnet_endpoints" {
+  default = []
+}
+
+resource "azurerm_subnet" "private" {
+  name                 = "${local.prefix}-private"
+  resource_group_name  = azurerm_resource_group.this.name
+  virtual_network_name = azurerm_virtual_network.this.name
+  address_prefixes     = [cidrsubnet(local.cidr, 3, 1)]
+
+  enforce_private_link_endpoint_network_policies = true
+  enforce_private_link_service_network_policies  = true
+
+  delegation {
+    name = "databricks"
+    service_delegation {
+      name = "Microsoft.Databricks/workspaces"
+      actions = [
+        "Microsoft.Network/virtualNetworks/subnets/join/action",
+        "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action",
+      "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action"]
+    }
+  }
+
+  service_endpoints = var.private_subnet_endpoints
+}
+
+resource "azurerm_subnet_network_security_group_association" "private" {
+  subnet_id                 = azurerm_subnet.private.id
+  network_security_group_id = azurerm_network_security_group.this.id
+}
+
+
+resource "azurerm_subnet" "plsubnet" {
+  name                                           = "${local.prefix}-privatelink"
+  resource_group_name                            = azurerm_resource_group.this.name
+  virtual_network_name                           = azurerm_virtual_network.this.name
+  address_prefixes                               = [cidrsubnet(local.cidr, 3, 2)]
+  enforce_private_link_endpoint_network_policies = true // set to true to disable subnet policy
+}
+
+
+resource "azurerm_virtual_network" "hubvnet" {
+  name                = "${local.prefix}-hub-vnet"
+  location            = azurerm_resource_group.this.location
+  resource_group_name = azurerm_resource_group.this.name
+  address_space       = [var.hubcidr]
+  tags                = local.tags
+}
+
+resource "azurerm_subnet" "hubfw" {
+  //name must be fixed as AzureFirewallSubnet
+  name                 = "AzureFirewallSubnet"
+  resource_group_name  = azurerm_resource_group.this.name
+  virtual_network_name = azurerm_virtual_network.hubvnet.name
+  address_prefixes     = [cidrsubnet(var.hubcidr, 3, 0)]
+}
+
+
+resource "azurerm_virtual_network_peering" "hubvnet" {
+  name                      = "peerhubtospoke"
+  resource_group_name       = azurerm_resource_group.this.name
+  virtual_network_name      = azurerm_virtual_network.hubvnet.name
+  remote_virtual_network_id = azurerm_virtual_network.this.id
+}
+
+resource "azurerm_virtual_network_peering" "spokevnet" {
+  name                      = "peerspoketohub"
+  resource_group_name       = azurerm_resource_group.this.name
+  virtual_network_name      = azurerm_virtual_network.this.name
+  remote_virtual_network_id = azurerm_virtual_network.hubvnet.id
+}

adb-external-hive-metastore/workspace.tf

@@ -0,0 +1,23 @@
+resource "azurerm_databricks_workspace" "this" {
+  name                                  = "${local.prefix}-workspace"
+  resource_group_name                   = azurerm_resource_group.this.name
+  location                              = azurerm_resource_group.this.location
+  sku                                   = "premium"
+  tags                                  = local.tags
+  customer_managed_key_enabled          = true
+  //infrastructure_encryption_enabled = true
+  custom_parameters {
+    no_public_ip                                         = var.no_public_ip
+    virtual_network_id                                   = azurerm_virtual_network.this.id
+    private_subnet_name                                  = azurerm_subnet.public.name
+    public_subnet_name                                   = azurerm_subnet.private.name
+    public_subnet_network_security_group_association_id  = azurerm_subnet_network_security_group_association.public.id
+    private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private.id
+    storage_account_name                                 = local.dbfsname
+  }
+  # We need this, otherwise destroy doesn't cleanup things correctly
+  depends_on = [
+    azurerm_subnet_network_security_group_association.public,
+    azurerm_subnet_network_security_group_association.private
+  ]
+}