From cf247faec73a07d2ef919fba6f04fc22f06ac599 Mon Sep 17 00:00:00 2001 From: Jocelyne Date: Mon, 8 Apr 2024 13:02:07 +0200 Subject: [PATCH] feat: Add migrate function --- exposed-core/api/exposed-core.api | 10 ++ exposed-core/build.gradle.kts | 4 + .../org/jetbrains/exposed/sql/Database.kt | 159 ++++++++++++++++++ .../org/jetbrains/exposed/sql/Exceptions.kt | 3 + .../org/jetbrains/exposed/sql/SchemaUtils.kt | 24 ++- .../shared/ddl/DatabaseMigrationTests.kt | 107 ++++++++++++ gradle/libs.versions.toml | 6 + 7 files changed, 312 insertions(+), 1 deletion(-) diff --git a/exposed-core/api/exposed-core.api b/exposed-core/api/exposed-core.api index 090fc31d6e..73f1dd8a57 100644 --- a/exposed-core/api/exposed-core.api +++ b/exposed-core/api/exposed-core.api @@ -73,6 +73,10 @@ public final class org/jetbrains/exposed/exceptions/DuplicateColumnException : j public fun (Ljava/lang/String;Ljava/lang/String;)V } +public final class org/jetbrains/exposed/exceptions/ExposedMigrationException : java/lang/RuntimeException { + public fun (Ljava/lang/Exception;Ljava/lang/String;)V +} + public final class org/jetbrains/exposed/exceptions/ExposedSQLException : java/sql/SQLException { public fun (Ljava/lang/Throwable;Ljava/util/List;Lorg/jetbrains/exposed/sql/Transaction;)V public final fun causedByQueries ()Ljava/util/List; @@ -589,6 +593,10 @@ public final class org/jetbrains/exposed/sql/Database { public final fun getVendor ()Ljava/lang/String; public final fun getVersion ()Ljava/math/BigDecimal; public final fun isVersionCovers (Ljava/math/BigDecimal;)Z + public final fun migrate ([Lorg/jetbrains/exposed/sql/Table;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Z)V + public final fun migrate ([Lorg/jetbrains/exposed/sql/Table;Ljavax/sql/DataSource;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Z)V + public static synthetic fun migrate$default (Lorg/jetbrains/exposed/sql/Database;[Lorg/jetbrains/exposed/sql/Table;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZILjava/lang/Object;)V + public static synthetic fun migrate$default (Lorg/jetbrains/exposed/sql/Database;[Lorg/jetbrains/exposed/sql/Table;Ljavax/sql/DataSource;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZILjava/lang/Object;)V public final fun setUseNestedTransactions (Z)V public fun toString ()Ljava/lang/String; } @@ -1963,7 +1971,9 @@ public final class org/jetbrains/exposed/sql/SchemaUtils { public static synthetic fun dropSchema$default (Lorg/jetbrains/exposed/sql/SchemaUtils;[Lorg/jetbrains/exposed/sql/Schema;ZZILjava/lang/Object;)V public final fun dropSequence ([Lorg/jetbrains/exposed/sql/Sequence;Z)V public static synthetic fun dropSequence$default (Lorg/jetbrains/exposed/sql/SchemaUtils;[Lorg/jetbrains/exposed/sql/Sequence;ZILjava/lang/Object;)V + public final fun generateMigrationScript ([Lorg/jetbrains/exposed/sql/Table;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Z)Ljava/io/File; public final fun generateMigrationScript ([Lorg/jetbrains/exposed/sql/Table;Ljava/lang/String;Ljava/lang/String;Z)Ljava/io/File; + public static synthetic fun generateMigrationScript$default (Lorg/jetbrains/exposed/sql/SchemaUtils;[Lorg/jetbrains/exposed/sql/Table;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZILjava/lang/Object;)Ljava/io/File; public static synthetic fun generateMigrationScript$default (Lorg/jetbrains/exposed/sql/SchemaUtils;[Lorg/jetbrains/exposed/sql/Table;Ljava/lang/String;Ljava/lang/String;ZILjava/lang/Object;)Ljava/io/File; public final fun listDatabases ()Ljava/util/List; public final fun listTables ()Ljava/util/List; diff --git a/exposed-core/build.gradle.kts b/exposed-core/build.gradle.kts index e25c77fd89..559bbede35 100644 --- a/exposed-core/build.gradle.kts +++ b/exposed-core/build.gradle.kts @@ -15,4 +15,8 @@ dependencies { api(kotlin("reflect")) api(libs.kotlinx.coroutines) api(libs.slf4j) + api(libs.flyway) + api(libs.flyway.mysql) + api(libs.flyway.oracle) + api(libs.flyway.sqlserver) } diff --git a/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/Database.kt b/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/Database.kt index a13c1e793b..56e40f01d3 100644 --- a/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/Database.kt +++ b/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/Database.kt @@ -1,11 +1,16 @@ package org.jetbrains.exposed.sql +import org.flywaydb.core.Flyway +import org.flywaydb.core.api.FlywayException +import org.flywaydb.core.api.output.MigrateResult import org.jetbrains.annotations.TestOnly +import org.jetbrains.exposed.exceptions.ExposedMigrationException import org.jetbrains.exposed.sql.statements.api.ExposedConnection import org.jetbrains.exposed.sql.statements.api.ExposedDatabaseMetadata import org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManager import org.jetbrains.exposed.sql.transactions.TransactionManager import org.jetbrains.exposed.sql.vendors.* +import java.io.File import java.math.BigDecimal import java.sql.Connection import java.sql.DriverManager @@ -104,6 +109,160 @@ class Database private constructor( */ internal var dataSourceReadOnly: Boolean = false + /** + * @param tables The tables to which the migration will be applied. + * @param user The user of the database. + * @param password The password of the database. + * @param oldVersion The version to migrate from. Pending migrations up to [oldVersion] are applied before applying the migration to [newVersion]. + * @param newVersion The version to migrate to. + * @param migrationTitle The title of the migration. + * @param migrationScriptDirectory The directory in which to create the migration script. + * @param withLogs By default, a description for each intermediate step, as well as its execution time, is logged at + * the INFO level. This can be disabled by setting [withLogs] to `false`. + * + * @throws ExposedMigrationException if the migration fails. + * + * Applies a database migration from [oldVersion] to [newVersion]. + * + * If a migration script with the same name already exists, the existing one will be used as is and a new one will + * not be generated. This allows you to generate a migration script before the migration and modify it manually if + * needed. + */ + @ExperimentalDatabaseMigrationApi + @Suppress("LongParameterList", "TooGenericExceptionCaught") + fun migrate( + vararg tables: Table, + user: String, + password: String, + oldVersion: String, + newVersion: String, + migrationTitle: String, + migrationScriptDirectory: String, + withLogs: Boolean = true + ) { + val flyway = Flyway + .configure() + .baselineOnMigrate(true) + .baselineVersion(oldVersion) + .dataSource(url, user, password) + .locations("filesystem:$migrationScriptDirectory") + .load() + + with(TransactionManager.current()) { + db.dialect.resetCaches() + + try { + val migrationScript = File("$migrationScriptDirectory/$migrationTitle.sql") + if (!migrationScript.exists()) { + SchemaUtils.generateMigrationScript( + tables = *tables, + newVersion = newVersion, + title = migrationTitle, + scriptDirectory = migrationScriptDirectory, + withLogs = withLogs + ) + } + } catch (exception: Exception) { + throw ExposedMigrationException( + exception = exception, + message = "Failed to generate migration script for migration from $oldVersion to $newVersion: ${exception.message.orEmpty()}" + ) + } + + try { + SchemaUtils.logTimeSpent("Migrating database from $oldVersion to $newVersion", withLogs = true) { + val migrateResult: MigrateResult = flyway.migrate() + if (withLogs) { + exposedLogger.info("Migration of database ${if (migrateResult.success) "succeeded" else "failed"}.") + } + } + } catch (exception: FlywayException) { + flyway.repair() + throw ExposedMigrationException( + exception = exception, + message = "Migration failed from version $oldVersion to $newVersion: ${exception.message.orEmpty()}" + ) + } + + db.dialect.resetCaches() + } + } + + /** + * @param tables The tables to which the migration will be applied. + * @param dataSource The [DataSource] object to be used as a means of getting a connection. + * @param oldVersion The version to migrate from. Pending migrations up to [oldVersion] are applied before applying the migration to [newVersion]. + * @param newVersion The version to migrate to. + * @param migrationTitle The title of the migration. + * @param migrationScriptDirectory The directory in which to create the migration script. + * @param withLogs By default, a description for each intermediate step, as well as its execution time, is logged at + * the INFO level. This can be disabled by setting [withLogs] to `false`. + * + * @throws ExposedMigrationException if the migration fails. + * + * Applies a database migration from [oldVersion] to [newVersion]. + * For PostgreSQLNG, "jdbc:pgsql" in the database URL is replaced with "jdbc:postgresql" because the former is not + * supported by Flyway. + */ + @ExperimentalDatabaseMigrationApi + @Suppress("LongParameterList", "TooGenericExceptionCaught") + fun migrate( + vararg tables: Table, + dataSource: DataSource, + oldVersion: String, + newVersion: String, + migrationTitle: String, + migrationScriptDirectory: String, + withLogs: Boolean = true + ) { + val flyway = Flyway + .configure() + .baselineOnMigrate(true) + .baselineVersion(oldVersion) + .dataSource(dataSource) + .locations("filesystem:$migrationScriptDirectory") + .load() + + with(TransactionManager.current()) { + db.dialect.resetCaches() + + try { + val migrationScript = File("$migrationScriptDirectory/$migrationTitle.sql") + if (!migrationScript.exists()) { + SchemaUtils.generateMigrationScript( + tables = *tables, + newVersion = newVersion, + title = migrationTitle, + scriptDirectory = migrationScriptDirectory, + withLogs = withLogs + ) + } + } catch (exception: Exception) { + throw ExposedMigrationException( + exception = exception, + message = "Failed to generate migration script for migration from $oldVersion to $newVersion: ${exception.message.orEmpty()}" + ) + } + + try { + SchemaUtils.logTimeSpent("Migrating database from $oldVersion to $newVersion", withLogs = true) { + val migrateResult: MigrateResult = flyway.migrate() + if (withLogs) { + exposedLogger.info("Migration of database ${if (migrateResult.success) "succeeded" else "failed"}.") + } + } + } catch (exception: FlywayException) { + flyway.repair() + throw ExposedMigrationException( + exception = exception, + message = "Migration failed from version $oldVersion to $newVersion: ${exception.message.orEmpty()}" + ) + } + + db.dialect.resetCaches() + } + } + companion object { internal val dialects = ConcurrentHashMap DatabaseDialect>() diff --git a/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/Exceptions.kt b/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/Exceptions.kt index 08f68ba960..c6f329d231 100644 --- a/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/Exceptions.kt +++ b/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/Exceptions.kt @@ -77,3 +77,6 @@ internal fun Transaction.throwUnsupportedException(message: String): Nothing = t message, db.dialect ) + +/** An exception thrown when a database migration fails. */ +class ExposedMigrationException(exception: Exception, message: String) : RuntimeException(message, exception) diff --git a/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/SchemaUtils.kt b/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/SchemaUtils.kt index eff1c97f31..93b0d9e952 100644 --- a/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/SchemaUtils.kt +++ b/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/SchemaUtils.kt @@ -10,7 +10,7 @@ import java.math.BigDecimal /** Utility functions that assist with creating, altering, and dropping database schema objects. */ @Suppress("TooManyFunctions", "LargeClass") object SchemaUtils { - private inline fun logTimeSpent(message: String, withLogs: Boolean, block: () -> R): R { + internal inline fun logTimeSpent(message: String, withLogs: Boolean, block: () -> R): R { return if (withLogs) { val start = System.currentTimeMillis() val answer = block() @@ -831,6 +831,28 @@ object SchemaUtils { return toDrop.toList() } + /** + * @param tables The tables whose changes will be used to generate the migration script. + * @param newVersion The version to migrate to. + * @param title The title of the migration. + * @param scriptDirectory The directory in which to create the migration script. + * @param withLogs By default, a description for each intermediate step, as well as its execution time, is logged at + * the INFO level. This can be disabled by setting [withLogs] to `false`. + * + * @return The generated migration script. + * + * @throws IllegalArgumentException if no argument is passed for the [tables] parameter. + * + * This function simply generates the migration script without applying the migration. The purpose of it is to show + * the user what the migration script will look like before applying the migration. + * This function uses the Flyway naming convention when generating the migration script. + * If a migration script with the same name already exists, its content will be overwritten. + */ + @ExperimentalDatabaseMigrationApi + fun generateMigrationScript(vararg tables: Table, newVersion: String, title: String, scriptDirectory: String, withLogs: Boolean = true): File { + return generateMigrationScript(*tables, scriptName = "V${newVersion}__$title", scriptDirectory = scriptDirectory, withLogs = withLogs) + } + /** * @param tables The tables whose changes will be used to generate the migration script. * @param scriptName The name to be used for the generated migration script. diff --git a/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/ddl/DatabaseMigrationTests.kt b/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/ddl/DatabaseMigrationTests.kt index c9f0956a1d..fb069bb061 100644 --- a/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/ddl/DatabaseMigrationTests.kt +++ b/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/ddl/DatabaseMigrationTests.kt @@ -1,5 +1,7 @@ package org.jetbrains.exposed.sql.tests.shared.ddl +import com.impossibl.postgres.jdbc.PGDataSource +import org.jetbrains.exposed.sql.Database import org.jetbrains.exposed.sql.ExperimentalDatabaseMigrationApi import org.jetbrains.exposed.sql.SchemaUtils import org.jetbrains.exposed.sql.Table @@ -12,7 +14,9 @@ import org.jetbrains.exposed.sql.tests.shared.assertEqualLists import org.jetbrains.exposed.sql.tests.shared.assertEquals import org.jetbrains.exposed.sql.tests.shared.assertTrue import org.jetbrains.exposed.sql.tests.shared.expectException +import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.vendors.PrimaryKeyMetadata +import org.junit.Assume import org.junit.Test import java.io.File import kotlin.properties.Delegates @@ -247,4 +251,107 @@ class DatabaseMigrationTests : DatabaseTestsBase() { } } } + + @Test + fun testMigration() { + val testTableWithoutIndex = object : Table("tester") { + val id = integer("id") + val name = varchar("name", length = 42) + + override val primaryKey = PrimaryKey(id) + } + + val testTableWithIndex = object : Table("tester") { + val id = integer("id") + val name = varchar("name", length = 42) + + override val primaryKey = PrimaryKey(id) + val byName = index("test_table_by_name", false, name) + } + + val migrationTitle = "AddIndex" + val migrationScriptDirectory = "src/test/resources" + + withDb(excludeSettings = listOf(TestDB.POSTGRESQLNG)) { + if (!isOldMySql()) { + try { + SchemaUtils.create(testTableWithoutIndex) + assertTrue(testTableWithoutIndex.exists()) + + val database = it.db!! + database.migrate( + testTableWithIndex, + user = it.user, + password = it.pass, + oldVersion = "1", + newVersion = "2.0", + migrationTitle = migrationTitle, + migrationScriptDirectory = migrationScriptDirectory + ) + + assertEquals(1, currentDialectTest.existingIndices(testTableWithoutIndex).size) + } finally { + SchemaUtils.drop(testTableWithoutIndex) + assertTrue(File("$migrationScriptDirectory/V2.0__$migrationTitle.sql").delete()) + } + } + } + } + + @Test + fun testPostgreSQLNGMigration() { + Assume.assumeTrue(TestDB.POSTGRESQLNG in TestDB.enabledDialects()) + + val testTableWithoutIndex = object : Table("tester") { + val id = integer("id") + val name = varchar("name", length = 42) + + override val primaryKey = PrimaryKey(id) + } + + val testTableWithIndex = object : Table("tester") { + val id = integer("id") + val name = varchar("name", length = 42) + + override val primaryKey = PrimaryKey(id) + val byName = index("test_table_by_name", false, name) + } + + val schemaHistoryTable = object : Table("flyway_schema_history") {} + + val migrationTitle = "AddIndex" + val migrationScriptDirectory = "src/test/resources" + + val dataSource = PGDataSource().apply { + url = "jdbc:pgsql://127.0.0.1:3004/postgres?lc_messages=en_US.UTF-8" + user = "root" + password = "Exposed_password_1!" + } + + val database = Database.connect(dataSource) + + transaction(database) { + try { + SchemaUtils.drop(schemaHistoryTable) + + SchemaUtils.create(testTableWithoutIndex) + assertTrue(testTableWithoutIndex.exists()) + + database.migrate( + testTableWithIndex, + dataSource = dataSource, + oldVersion = "1", + newVersion = "2.0", + migrationTitle = migrationTitle, + migrationScriptDirectory = migrationScriptDirectory + ) + + assertEquals(1, currentDialectTest.existingIndices(testTableWithoutIndex).size) + } finally { + SchemaUtils.drop(testTableWithoutIndex) + SchemaUtils.drop(schemaHistoryTable) + assertTrue(File("$migrationScriptDirectory/V2.0__$migrationTitle.sql").delete()) + } + } + } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6bbbc36004..6bae7edb18 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -32,6 +32,7 @@ javax-money = "1.1" moneta = "1.4.4" hikariCP = "4.0.3" logcaptor = "2.9.2" +flyway = "9.22.3" [libraries] jvm = { group = "org.jetbrains.kotlin.jvm", name = "org.jetbrains.kotlin.jvm.gradle.plugin", version.ref = "kotlin" } @@ -81,6 +82,11 @@ mssql = { group = "com.microsoft.sqlserver", name = "mssql-jdbc", version.ref = logcaptor = { group = "io.github.hakky54", name = "logcaptor", version.ref = "logcaptor" } +flyway = { group = "org.flywaydb", name = "flyway-core", version.ref = "flyway" } +flyway-mysql = { group = "org.flywaydb", name = "flyway-mysql", version.ref = "flyway" } +flyway-oracle = { group = "org.flywaydb", name = "flyway-database-oracle", version.ref = "flyway" } +flyway-sqlserver = { group = "org.flywaydb", name = "flyway-sqlserver", version.ref = "flyway" } + [plugins] jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } detekt = { id = "io.gitlab.arturbosch.detekt", version.ref = "detekt" }