fix data databricks cluster instance type issue

HariGS-DB · Aug 30, 2022 · 32d9bc2 · 32d9bc2
1 parent a9ffe37
commit 32d9bc2
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 9 deletions.
diff --git a/adb-external-hive-metastore/variables.tf b/adb-external-hive-metastore/variables.tf
@@ -40,6 +40,12 @@ variable "db_password" {
   sensitive   = true
 }
 
+variable "node_type" {
+  description = "instance type"
+  type        = string
+  default     = "Standard_E8ds_v4"
+}
+
 variable "cold_start" {
   description = "if true, will spin up a cluster to download hive jars to dbfs"
   type        = bool

diff --git a/adb-external-hive-metastore/versions.tf b/adb-external-hive-metastore/versions.tf
@@ -2,7 +2,7 @@
 terraform {
   required_providers {
     databricks = {
-      source = "databricks/databricks"
+      source  = "databricks/databricks"
       version = "0.5.1"
     }
 

diff --git a/adb-external-hive-metastore/workspace.tf b/adb-external-hive-metastore/workspace.tf
@@ -31,10 +31,6 @@ provider "databricks" {
   host = azurerm_databricks_workspace.this.workspace_url
 }
 
-data "databricks_node_type" "smallest" {
-  local_disk = true
-}
-
 data "databricks_spark_version" "latest_lts" {
   long_term_support = true
   depends_on = [
@@ -46,11 +42,11 @@ resource "databricks_cluster" "coldstart" {
   count                   = var.cold_start ? 1 : 0
   cluster_name            = "coldstart_cluster"
   spark_version           = data.databricks_spark_version.latest_lts.id
-  node_type_id            = data.databricks_node_type.smallest.id
-  autotermination_minutes = 20
+  node_type_id            = var.node_type
+  autotermination_minutes = 30
   autoscale {
-    min_workers = 1
-    max_workers = 2
+    min_workers = 0
+    max_workers = 1
   }
 
   spark_conf = {