Skip to content

Commit

Permalink
refreshed external metastore template
Browse files Browse the repository at this point in the history
  • Loading branch information
@hwang-db
hwang-db committed May 12, 2022
1 parent 8c11097 commit cc662c4
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 34 deletions.
1 change: 1 addition & 0 deletions adb-external-hive-metastore/README.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# ADB workspace with external hive metastore

Credits to [email protected] and [email protected] for notebook logic for database initialization steps.
This architecture will be deployed:
![alt text](../charts/adb-external-hive-metastore.png?raw=true)

Expand Down
90 changes: 56 additions & 34 deletions adb-external-hive-metastore/coldstart/metastore_coldstart.py
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,47 @@
# Databricks notebook source
# MAGIC %md
# MAGIC
# MAGIC - Hive JDBC URL
# MAGIC `jdbc:sqlserver://bk-sqlserver.database.windows.net:1433;database=hive3;user=abc@bk-sqlserver;password={your_password_here};encrypt=true;trustServerCertificate=false;hostNameInCertificate=*.database.windows.net;loginTimeout=30;`
# uncomment below if you are to remove jars previously downloaded using this script
# dbutils.fs.rm('/tmp/hive',True)

# COMMAND ----------

# DBTITLE 1,Download hive and hadoop tools lib
# MAGIC %sh
# MAGIC wget https://archive.apache.org/dist/hadoop/common/hadoop-2.7.2/hadoop-2.7.2.tar.gz
# MAGIC tar -xvzf hadoop-2.7.2.tar.gz --directory /opt
# MAGIC wget https://archive.apache.org/dist/hive/hive-3.1.0/apache-hive-3.1.0-bin.tar.gz
# MAGIC tar -xvzf apache-hive-3.1.0-bin.tar.gz --directory /opt
# MAGIC export TARGET_HIVE_VERSION="3.1.0"
# MAGIC export TARGET_HADOOP_VERSION="2.7.2"
# MAGIC export TARGET_HIVE_HOME="/opt/apache-hive-${TARGET_HIVE_VERSION}-bin"
# MAGIC export TARGET_HADOOP_HOME="/opt/hadoop-${TARGET_HADOOP_VERSION}"
# MAGIC export JARS_DIRECTORY="/dbfs/tmp/hive/3-1-0/lib/"
# MAGIC
# MAGIC
# MAGIC if [ ! -d "$TARGET_HADOOP_HOME" ]; then
# MAGIC if [ ! -f hadoop-${TARGET_HADOOP_VERSION}.tar.gz ]; then
# MAGIC wget https://archive.apache.org/dist/hadoop/common/hadoop-${TARGET_HADOOP_VERSION}/hadoop-${TARGET_HADOOP_VERSION}.tar.gz
# MAGIC fi
# MAGIC tar -xvzf hadoop-${TARGET_HADOOP_VERSION}.tar.gz --directory /opt
# MAGIC fi
# MAGIC if [ ! -d "$TARGET_HIVE_HOME" ]; then
# MAGIC if [ ! -f apache-hive-${TARGET_HIVE_VERSION}-bin.tar.gz ]; then
# MAGIC wget https://archive.apache.org/dist/hive/hive-${TARGET_HIVE_VERSION}/apache-hive-${TARGET_HIVE_VERSION}-bin.tar.gz
# MAGIC fi
# MAGIC tar -xvzf apache-hive-${TARGET_HIVE_VERSION}-bin.tar.gz --directory /opt
# MAGIC fi
# MAGIC ## https://www.microsoft.com/en-us/download/details.aspx?id=11774
# MAGIC wget https://download.microsoft.com/download/0/2/A/02AAE597-3865-456C-AE7F-613F99F850A8/sqljdbc_6.0.8112.200_enu.tar.gz
# MAGIC tar -xvzf sqljdbc_6.0.8112.200_enu.tar.gz --directory /opt
# MAGIC cp /opt/sqljdbc_6.0/enu/jre8/sqljdbc42.jar /opt/apache-hive-3.1.0-bin/lib/sqljdbc42.jar

# COMMAND ----------

# MAGIC %sh
# MAGIC mkdir -p /dbfs/tmp/hive/3-1-0/lib/
# MAGIC cp -r /opt/apache-hive-3.1.0-bin/lib/. /dbfs/tmp/hive/3-1-0/lib/
# MAGIC cp -r /opt/hadoop-2.7.2/share/hadoop/common/lib/. /dbfs/tmp/hive/3-1-0/lib/
# MAGIC if [ ! -f sqljdbc_9.2.1.0_enu.tar.gz ]; then
# MAGIC wget https://download.microsoft.com/download/4/c/3/4c31fbc1-62cc-4a0b-932a-b38ca31cd410/sqljdbc_9.2.1.0_enu.tar.gz
# MAGIC fi
# MAGIC tar -xvzf sqljdbc_9.2.1.0_enu.tar.gz --directory /opt
# MAGIC if [ ! -f oauth2-oidc-sdk-9.4.jar ]; then
# MAGIC wget https://repo1.maven.org/maven2/com/nimbusds/oauth2-oidc-sdk/9.4/oauth2-oidc-sdk-9.4.jar
# MAGIC fi
# MAGIC if [ ! -f msal4j-1.10.0.jar ]; then
# MAGIC wget https://repo1.maven.org/maven2/com/microsoft/azure/msal4j/1.10.0/msal4j-1.10.0.jar
# MAGIC fi
# MAGIC cp oauth2-oidc-sdk-9.4.jar msal4j-1.10.0.jar /opt/sqljdbc_9.2/enu/mssql-jdbc-9.2.1.jre8.jar ${TARGET_HIVE_HOME}/lib/
# MAGIC
# MAGIC echo "Copying necessary jars into ${JARS_DIRECTORY}"
# MAGIC mkdir -p ${JARS_DIRECTORY}
# MAGIC cp -r ${TARGET_HIVE_HOME}/lib/. ${JARS_DIRECTORY}
# MAGIC cp -r ${TARGET_HADOOP_HOME}/share/hadoop/common/lib/. ${JARS_DIRECTORY}
# MAGIC ls -l ${JARS_DIRECTORY}

# COMMAND ----------

Expand All @@ -34,30 +53,33 @@

# COMMAND ----------

# DBTITLE 1,Initialize hive schema
# DBTITLE 1,Initialize hive schema, you must use HIVE_HOME and HADOOP_HOME env var
# MAGIC %sh
# MAGIC export HIVE_HOME="/opt/apache-hive-3.1.0-bin"
# MAGIC export HADOOP_HOME="/opt/hadoop-2.7.2"
# MAGIC export TARGET_HIVE_VERSION="3.1.0"
# MAGIC export TARGET_HADOOP_VERSION="2.7.2"
# MAGIC export TARGET_HIVE_HOME="/opt/apache-hive-${TARGET_HIVE_VERSION}-bin"
# MAGIC export TARGET_HADOOP_HOME="/opt/hadoop-${TARGET_HADOOP_VERSION}"
# MAGIC export SQLDB_DRIVER="com.microsoft.sqlserver.jdbc.SQLServerDriver"
# MAGIC
# MAGIC # uncomment following line to init schema
# MAGIC /opt/apache-hive-3.1.0-bin/bin/schematool -dbType mssql -url $HIVE_URL -passWord $HIVE_PASSWORD -userName $HIVE_USER -driver $SQLDB_DRIVER -initSchema

# COMMAND ----------

# MAGIC %sh
# MAGIC # validate that schema is initialized
# MAGIC /opt/apache-hive-3.1.0-bin/bin/schematool -dbType mssql -url $HIVE_URL -passWord $HIVE_PASSWORD -userName $HIVE_USER -driver $SQLDB_DRIVER -info
# MAGIC export HIVE_HOME=$TARGET_HIVE_HOME
# MAGIC export HADOOP_HOME=$TARGET_HADOOP_HOME
# MAGIC
# MAGIC $TARGET_HIVE_HOME/bin/schematool -dbType mssql -url $HIVE_URL -passWord $HIVE_PASSWORD -userName $HIVE_USER -driver $SQLDB_DRIVER -initSchema

# COMMAND ----------

# DBTITLE 1,Validate hive is initialized
# DBTITLE 1,Validate hive is initialized, you must use HIVE_HOME and HADOOP_HOME env var
# MAGIC %sh
# MAGIC export HIVE_HOME="/opt/apache-hive-3.1.0-bin"
# MAGIC export HADOOP_HOME="/opt/hadoop-2.7.2"
# MAGIC export TARGET_HIVE_VERSION="3.1.0"
# MAGIC export TARGET_HADOOP_VERSION="2.7.2"
# MAGIC export TARGET_HIVE_HOME="/opt/apache-hive-${TARGET_HIVE_VERSION}-bin"
# MAGIC export TARGET_HADOOP_HOME="/opt/hadoop-${TARGET_HADOOP_VERSION}"
# MAGIC export SQLDB_DRIVER="com.microsoft.sqlserver.jdbc.SQLServerDriver"
# MAGIC
# MAGIC /opt/apache-hive-3.1.0-bin/bin/schematool -dbType mssql -url $HIVE_URL -passWord $HIVE_PASSWORD -userName $HIVE_USER -driver $SQLDB_DRIVER -info
# MAGIC export HIVE_HOME=$TARGET_HIVE_HOME
# MAGIC export HADOOP_HOME=$TARGET_HADOOP_HOME
# MAGIC
# MAGIC $TARGET_HIVE_HOME/bin/schematool -dbType mssql -url $HIVE_URL -passWord $HIVE_PASSWORD -userName $HIVE_USER -driver $SQLDB_DRIVER -info

# COMMAND ----------

Expand Down
Binary file modified charts/adb-external-hive-metastore.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

0 comments on commit cc662c4

Please sign in to comment.