Skip to content

Commit 0d1590f

Browse files
hwang-dbhwang-db
committed
draft for e.h.metastore
1 parent 263af66 commit 0d1590f

File tree

8 files changed

+334
-0
lines changed

8 files changed

+334
-0
lines changed

‎adb-external-hive-metastore/.terraform.lock.hcl

Lines changed: 74 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎adb-external-hive-metastore/README.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# ADB workspace with external hive metastore
2+
3+
This template creates:
4+
1. SQL Server
5+
2. SQL database
6+
3. ADB workspace

‎adb-external-hive-metastore/main.tf

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/**
2+
* Azure Databricks workspace in custom VNet
3+
*
4+
* Module creates:
5+
* * Resource group with random prefix
6+
* * Tags, including `Owner`, which is taken from `az account show --query user`
7+
* * VNet with public and private subnet
8+
* * Databricks workspace
9+
* * External Hive Metastore for ADB workspace
10+
*/
11+
provider "azurerm" {
12+
features {}
13+
}
14+
15+
provider "random" {
16+
}
17+
18+
resource "random_string" "naming" {
19+
special = false
20+
upper = false
21+
length = 6
22+
}
23+
24+
data "azurerm_client_config" "current" {
25+
}
26+
27+
data "external" "me" {
28+
program = ["az", "account", "show", "--query", "user"]
29+
}
30+
31+
locals {
32+
// dltp - databricks labs terraform provider
33+
prefix = join("-", [var.workspace_prefix, "${random_string.naming.result}"])
34+
location = var.rglocation
35+
cidr = var.spokecidr
36+
dbfsname = join("", [var.dbfs_prefix, "${random_string.naming.result}"]) // dbfs name must not have special chars
37+
38+
// tags that are propagated down to all resources
39+
tags = {
40+
Environment = "Testing"
41+
Owner = lookup(data.external.me.result, "name")
42+
Epoch = random_string.naming.result
43+
}
44+
}
45+
46+
resource "azurerm_resource_group" "this" {
47+
name = "adb-dev-${local.prefix}-rg"
48+
location = local.location
49+
tags = local.tags
50+
}
51+
52+
output "arm_client_id" {
53+
value = data.azurerm_client_config.current.client_id
54+
}
55+
56+
output "arm_subscription_id" {
57+
value = data.azurerm_client_config.current.subscription_id
58+
}
59+
60+
output "arm_tenant_id" {
61+
value = data.azurerm_client_config.current.tenant_id
62+
}
63+
64+
output "azure_region" {
65+
value = local.location
66+
}
67+
68+
output "resource_group" {
69+
value = azurerm_resource_group.this.name
70+
}

‎adb-external-hive-metastore/outputs.tf

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
output "databricks_azure_workspace_resource_id" {
2+
// The ID of the Databricks Workspace in the Azure management plane.
3+
value = azurerm_databricks_workspace.this.id
4+
}
5+
6+
output "workspace_url" {
7+
// The workspace URL which is of the format 'adb-{workspaceId}.{random}.azuredatabricks.net'
8+
// this is not named as DATABRICKS_HOST, because it affect authentication
9+
value = "https://${azurerm_databricks_workspace.this.workspace_url}/"
10+
}

‎adb-external-hive-metastore/variables.tf

Whitespace-only changes.

‎adb-external-hive-metastore/versions.tf

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# versions.tf
2+
terraform {
3+
required_providers {
4+
databricks = {
5+
source = "databrickslabs/databricks"
6+
version = "0.3.10"
7+
}
8+
9+
azurerm = {
10+
source = "hashicorp/azurerm"
11+
version = "=2.83.0"
12+
}
13+
}
14+
}

‎adb-external-hive-metastore/vnet.tf

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
resource "azurerm_virtual_network" "this" {
2+
name = "${local.prefix}-vnet"
3+
location = azurerm_resource_group.this.location
4+
resource_group_name = azurerm_resource_group.this.name
5+
address_space = [local.cidr]
6+
tags = local.tags
7+
}
8+
9+
resource "azurerm_network_security_group" "this" {
10+
name = "${local.prefix}-nsg"
11+
location = azurerm_resource_group.this.location
12+
resource_group_name = azurerm_resource_group.this.name
13+
tags = local.tags
14+
}
15+
16+
resource "azurerm_network_security_rule" "aad" {
17+
name = "AllowAAD"
18+
priority = 200
19+
direction = "Outbound"
20+
access = "Allow"
21+
protocol = "Tcp"
22+
source_port_range = "*"
23+
destination_port_range = "443"
24+
source_address_prefix = "VirtualNetwork"
25+
destination_address_prefix = "AzureActiveDirectory"
26+
resource_group_name = azurerm_resource_group.this.name
27+
network_security_group_name = azurerm_network_security_group.this.name
28+
}
29+
30+
resource "azurerm_network_security_rule" "azfrontdoor" {
31+
name = "AllowAzureFrontDoor"
32+
priority = 201
33+
direction = "Outbound"
34+
access = "Allow"
35+
protocol = "Tcp"
36+
source_port_range = "*"
37+
destination_port_range = "443"
38+
source_address_prefix = "VirtualNetwork"
39+
destination_address_prefix = "AzureFrontDoor.Frontend"
40+
resource_group_name = azurerm_resource_group.this.name
41+
network_security_group_name = azurerm_network_security_group.this.name
42+
}
43+
resource "azurerm_subnet" "public" {
44+
name = "${local.prefix}-public"
45+
resource_group_name = azurerm_resource_group.this.name
46+
virtual_network_name = azurerm_virtual_network.this.name
47+
address_prefixes = [cidrsubnet(local.cidr, 3, 0)]
48+
49+
delegation {
50+
name = "databricks"
51+
service_delegation {
52+
name = "Microsoft.Databricks/workspaces"
53+
actions = [
54+
"Microsoft.Network/virtualNetworks/subnets/join/action",
55+
"Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action",
56+
"Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action"]
57+
}
58+
}
59+
}
60+
61+
resource "azurerm_subnet_network_security_group_association" "public" {
62+
subnet_id = azurerm_subnet.public.id
63+
network_security_group_id = azurerm_network_security_group.this.id
64+
}
65+
66+
variable "private_subnet_endpoints" {
67+
default = []
68+
}
69+
70+
resource "azurerm_subnet" "private" {
71+
name = "${local.prefix}-private"
72+
resource_group_name = azurerm_resource_group.this.name
73+
virtual_network_name = azurerm_virtual_network.this.name
74+
address_prefixes = [cidrsubnet(local.cidr, 3, 1)]
75+
76+
enforce_private_link_endpoint_network_policies = true
77+
enforce_private_link_service_network_policies = true
78+
79+
delegation {
80+
name = "databricks"
81+
service_delegation {
82+
name = "Microsoft.Databricks/workspaces"
83+
actions = [
84+
"Microsoft.Network/virtualNetworks/subnets/join/action",
85+
"Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action",
86+
"Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action"]
87+
}
88+
}
89+
90+
service_endpoints = var.private_subnet_endpoints
91+
}
92+
93+
resource "azurerm_subnet_network_security_group_association" "private" {
94+
subnet_id = azurerm_subnet.private.id
95+
network_security_group_id = azurerm_network_security_group.this.id
96+
}
97+
98+
99+
resource "azurerm_subnet" "plsubnet" {
100+
name = "${local.prefix}-privatelink"
101+
resource_group_name = azurerm_resource_group.this.name
102+
virtual_network_name = azurerm_virtual_network.this.name
103+
address_prefixes = [cidrsubnet(local.cidr, 3, 2)]
104+
enforce_private_link_endpoint_network_policies = true // set to true to disable subnet policy
105+
}
106+
107+
108+
resource "azurerm_virtual_network" "hubvnet" {
109+
name = "${local.prefix}-hub-vnet"
110+
location = azurerm_resource_group.this.location
111+
resource_group_name = azurerm_resource_group.this.name
112+
address_space = [var.hubcidr]
113+
tags = local.tags
114+
}
115+
116+
resource "azurerm_subnet" "hubfw" {
117+
//name must be fixed as AzureFirewallSubnet
118+
name = "AzureFirewallSubnet"
119+
resource_group_name = azurerm_resource_group.this.name
120+
virtual_network_name = azurerm_virtual_network.hubvnet.name
121+
address_prefixes = [cidrsubnet(var.hubcidr, 3, 0)]
122+
}
123+
124+
125+
resource "azurerm_virtual_network_peering" "hubvnet" {
126+
name = "peerhubtospoke"
127+
resource_group_name = azurerm_resource_group.this.name
128+
virtual_network_name = azurerm_virtual_network.hubvnet.name
129+
remote_virtual_network_id = azurerm_virtual_network.this.id
130+
}
131+
132+
resource "azurerm_virtual_network_peering" "spokevnet" {
133+
name = "peerspoketohub"
134+
resource_group_name = azurerm_resource_group.this.name
135+
virtual_network_name = azurerm_virtual_network.this.name
136+
remote_virtual_network_id = azurerm_virtual_network.hubvnet.id
137+
}

‎adb-external-hive-metastore/workspace.tf

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
resource "azurerm_databricks_workspace" "this" {
2+
name = "${local.prefix}-workspace"
3+
resource_group_name = azurerm_resource_group.this.name
4+
location = azurerm_resource_group.this.location
5+
sku = "premium"
6+
tags = local.tags
7+
customer_managed_key_enabled = true
8+
//infrastructure_encryption_enabled = true
9+
custom_parameters {
10+
no_public_ip = var.no_public_ip
11+
virtual_network_id = azurerm_virtual_network.this.id
12+
private_subnet_name = azurerm_subnet.public.name
13+
public_subnet_name = azurerm_subnet.private.name
14+
public_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.public.id
15+
private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.private.id
16+
storage_account_name = local.dbfsname
17+
}
18+
# We need this, otherwise destroy doesn't cleanup things correctly
19+
depends_on = [
20+
azurerm_subnet_network_security_group_association.public,
21+
azurerm_subnet_network_security_group_association.private
22+
]
23+
}

0 commit comments

Comments
 (0)