diff --git a/.gitignore b/.gitignore index c9bb642..358bdee 100644 --- a/.gitignore +++ b/.gitignore @@ -352,4 +352,4 @@ MigrationBackup/ project/project/ project/target/ target/ - +.idea diff --git a/build.sbt b/build.sbt deleted file mode 100644 index 5425a00..0000000 --- a/build.sbt +++ /dev/null @@ -1,32 +0,0 @@ -name := "spark-mssql-connector" - -organization := "com.microsoft.sqlserver.jdbc.spark" - -version := "1.0.0" - -scalaVersion := "2.11.12" - -val sparkVersion = "2.4.6" - -javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint") - -libraryDependencies ++= Seq( - "org.apache.spark" %% "spark-sql" % sparkVersion % "provided", - // Spark Testing Utilities - "org.apache.spark" %% "spark-core" % sparkVersion % "test" classifier - "tests", - "org.apache.spark" %% "spark-sql" % sparkVersion% "test" classifier - "tests", - "org.apache.spark" %% "spark-catalyst" % sparkVersion % "test" classifier - "tests", - "org.scalatest" %% "scalatest" % "3.0.5" % "test", - "com.novocode" % "junit-interface" % "0.11" % "test", - - //SQLServer JDBC jars - "com.microsoft.sqlserver" % "mssql-jdbc" % "8.4.1.jre8" -) - -scalacOptions := Seq("-unchecked", "-deprecation", "evicted") - -// Exclude scala-library from this fat jar. The scala library is already there in spark package. -assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false) diff --git a/pom.xml b/pom.xml index f899490..d14c735 100644 --- a/pom.xml +++ b/pom.xml @@ -19,9 +19,6 @@ UTF-8 - 2.11 - 2.11.12 - 2.4.6 @@ -56,23 +53,12 @@ test tests - - org.scalatest - scalatest_${scala.binary.version} - 3.0.5 - test - com.novocode junit-interface 0.11 test - - com.microsoft.sqlserver - mssql-jdbc - 8.4.1.jre8 - @@ -213,4 +199,48 @@ + + + spark24 + + 2.11 + 2.11.12 + 2.4.6 + + + + org.scalatest + scalatest_${scala.binary.version} + 3.0.5 + test + + + com.microsoft.sqlserver + mssql-jdbc + 8.4.1.jre8 + + + + + spark30 + + 2.12 + 2.12.11 + 3.0.0 + + + + org.scalatest + scalatest_${scala.binary.version} + 3.0.8 + test + + + com.microsoft.sqlserver + mssql-jdbc + 8.4.1.jre8 + + + + diff --git a/src/test/scala/com/microsoft/sqlserver/jdbc/spark/bulkwrite/DataSourceTest.scala b/src/test/scala/com/microsoft/sqlserver/jdbc/spark/bulkwrite/DataSourceTest.scala index 2d900bc..d033cfc 100644 --- a/src/test/scala/com/microsoft/sqlserver/jdbc/spark/bulkwrite/DataSourceTest.scala +++ b/src/test/scala/com/microsoft/sqlserver/jdbc/spark/bulkwrite/DataSourceTest.scala @@ -16,9 +16,9 @@ import java.sql.Connection import org.scalatest.Matchers import org.apache.spark.SparkFunSuite -import org.apache.spark.sql.test.SharedSQLContext +import org.apache.spark.sql.test.SharedSparkSession -class DataSourceTest extends SparkFunSuite with Matchers with SharedSQLContext { +class DataSourceTest extends SparkFunSuite with Matchers with SharedSparkSession { test("Schema validation between Spark DataFrame and SQL Server ResultSet") {}