From 11a40c5b28d2a8299b359a73c8052be48e27a41c Mon Sep 17 00:00:00 2001 From: Stephane Geneix Date: Wed, 18 Sep 2024 07:42:44 -0700 Subject: [PATCH] destination-e2e: upgrade CDK --- .../destination/DestinationAcceptanceTest.kt | 12 +++++------ .../destination-dev-null/build.gradle | 4 ++-- .../destination-dev-null/metadata.yaml | 2 +- .../DevNullDestinationAcceptanceTest.kt | 21 +++++++------------ .../destination-e2e-test/build.gradle | 4 ++-- .../destination-e2e-test/metadata.yaml | 2 +- .../e2e_test/TestingDestinations.kt | 4 ++-- .../e2e_test/ThrottledDestination.kt | 2 +- .../TestingSilentDestinationAcceptanceTest.kt | 10 ++++----- .../e2e_test/ThrottledDestinationTest.kt | 3 ++- docs/integrations/destinations/e2e-test.md | 3 ++- 11 files changed, 30 insertions(+), 37 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt index a14b9a7b12ef..fcaffe850012 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt @@ -193,7 +193,7 @@ abstract class DestinationAcceptanceTest( */ @Throws(Exception::class) protected abstract fun retrieveRecords( - testEnv: TestDestinationEnv?, + testEnv: TestDestinationEnv, streamName: String, namespace: String, streamSchema: JsonNode @@ -236,7 +236,7 @@ abstract class DestinationAcceptanceTest( } private fun retrieveRecordsDataOnly( - testEnv: TestDestinationEnv?, + testEnv: TestDestinationEnv, streamName: String, namespace: String, streamSchema: JsonNode @@ -1040,7 +1040,7 @@ abstract class DestinationAcceptanceTest( getProtocolVersion() ) ) - .lines() + .lines().filter { it.isNotEmpty() } .map { Jsons.deserialize(it, AirbyteMessage::class.java) } val config = getConfig() runSyncAndVerifyStateOutput( @@ -1486,7 +1486,7 @@ abstract class DestinationAcceptanceTest( */ @Test @Throws(Exception::class) - fun testSyncNotFailsWithNewFields() { + open fun testSyncNotFailsWithNewFields() { if (!implementsOverwrite()) { LOGGER.info { "Destination's spec.json does not support overwrite sync mode." } return @@ -1724,7 +1724,7 @@ abstract class DestinationAcceptanceTest( } @Test - fun testAirbyteTimeTypes() { + open fun testAirbyteTimeTypes() { val configuredCatalog = Jsons.deserialize( MoreResources.readResource("v0/every_time_type_configured_catalog.json"), @@ -2171,7 +2171,7 @@ abstract class DestinationAcceptanceTest( } // ignores emitted at. - protected fun assertSameMessages( + open protected fun assertSameMessages( expected: List, actual: List, pruneAirbyteInternalFields: Boolean diff --git a/airbyte-integrations/connectors/destination-dev-null/build.gradle b/airbyte-integrations/connectors/destination-dev-null/build.gradle index e4a9dfb1d487..dff97c133d29 100644 --- a/airbyte-integrations/connectors/destination-dev-null/build.gradle +++ b/airbyte-integrations/connectors/destination-dev-null/build.gradle @@ -3,9 +3,9 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.20.6' + cdkVersionRequired = '0.45.0' features = ['db-destinations'] - useLocalCdk = false + useLocalCdk = true } application { diff --git a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml index 8185344dff5a..d01d46fd0def 100644 --- a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml +++ b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: a7bcc9d8-13b3-4e49-b80d-d020b90045e3 - dockerImageTag: 0.4.1 + dockerImageTag: 0.5.0 dockerRepository: airbyte/destination-dev-null githubIssueLabel: destination-dev-null icon: airbyte.svg diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.kt b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.kt index cf78fb8a1e11..a87ac5012e90 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.kt @@ -14,9 +14,7 @@ import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test class DevNullDestinationAcceptanceTest : DestinationAcceptanceTest() { - override fun getImageName(): String { - return "airbyte/destination-dev-null:dev" - } + override val imageName = "airbyte/destination-dev-null:dev" override fun getConfig(): JsonNode { return Jsons.jsonNode( @@ -36,12 +34,7 @@ class DevNullDestinationAcceptanceTest : DestinationAcceptanceTest() { ) } - override fun retrieveRecords( - testEnv: TestDestinationEnv, - streamName: String, - namespace: String?, - streamSchema: JsonNode - ): List { + override fun retrieveRecords(testEnv: TestDestinationEnv, streamName: String, namespace: String, streamSchema: JsonNode): List { return emptyList() } @@ -64,12 +57,12 @@ class DevNullDestinationAcceptanceTest : DestinationAcceptanceTest() { // Skip because `retrieveRecords` returns an empty list at all times. @Disabled @Test override fun testSyncNotFailsWithNewFields() {} - // This test assumes that dedup support means normalization support. - // Override it to do nothing. @Disabled @Test - @Throws(Exception::class) - override fun testIncrementalDedupeSync() { - super.testIncrementalDedupeSync() + override fun testAirbyteTimeTypes() { + } + + open override fun getDefaultSchema(config: JsonNode): String? { + return super.getDefaultSchema(config) ?: "default_schema" } } diff --git a/airbyte-integrations/connectors/destination-e2e-test/build.gradle b/airbyte-integrations/connectors/destination-e2e-test/build.gradle index c1a95d154e3b..c3006ff9213c 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/build.gradle +++ b/airbyte-integrations/connectors/destination-e2e-test/build.gradle @@ -3,9 +3,9 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.20.6' + cdkVersionRequired = '0.45.0' features = ['db-destinations'] - useLocalCdk = false + useLocalCdk = true } application { diff --git a/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml b/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml index 8b2fa36f25fb..ff58dea88f1d 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml +++ b/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: unknown connectorType: destination definitionId: 2eb65e87-983a-4fd7-b3e3-9d9dc6eb8537 - dockerImageTag: 0.4.1 + dockerImageTag: 0.5.0 dockerRepository: airbyte/destination-e2e-test githubIssueLabel: destination-e2e-test icon: airbyte.svg diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/TestingDestinations.kt b/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/TestingDestinations.kt index bcd3d5cc18c1..4fe45119e713 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/TestingDestinations.kt +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/TestingDestinations.kt @@ -47,12 +47,12 @@ constructor( config: JsonNode, catalog: ConfiguredAirbyteCatalog, outputRecordCollector: Consumer - ): AirbyteMessageConsumer { + ): AirbyteMessageConsumer? { return selectDestination(config)!!.getConsumer(config, catalog, outputRecordCollector) } @Throws(Exception::class) - override fun check(config: JsonNode): AirbyteConnectionStatus { + override fun check(config: JsonNode): AirbyteConnectionStatus? { return try { selectDestination(config)!!.check(config) } catch (e: Exception) { diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.kt b/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.kt index 2ccf3688236b..7d95fc956ffc 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.kt +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.kt @@ -25,7 +25,7 @@ class ThrottledDestination : BaseConnector(), Destination { override fun getConsumer( config: JsonNode, - catalog: ConfiguredAirbyteCatalog?, + catalog: ConfiguredAirbyteCatalog, outputRecordCollector: Consumer ): AirbyteMessageConsumer { return ThrottledConsumer( diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/kotlin/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.kt b/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/kotlin/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.kt index 82399bd0ce1f..435eb40cee44 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/kotlin/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.kt +++ b/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/kotlin/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.kt @@ -14,9 +14,7 @@ import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test class TestingSilentDestinationAcceptanceTest : DestinationAcceptanceTest() { - override fun getImageName(): String { - return "airbyte/destination-e2e-test:dev" - } + override val imageName = "airbyte/destination-e2e-test:dev" override fun getConfig(): JsonNode { return Jsons.jsonNode( @@ -38,9 +36,9 @@ class TestingSilentDestinationAcceptanceTest : DestinationAcceptanceTest() { override fun retrieveRecords( testEnv: TestDestinationEnv?, - streamName: String?, - namespace: String?, - streamSchema: JsonNode? + streamName: String, + namespace: String, + streamSchema: JsonNode ): List { return emptyList() } diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/test/kotlin/io/airbyte/integrations/destination/e2e_test/ThrottledDestinationTest.kt b/airbyte-integrations/connectors/destination-e2e-test/src/test/kotlin/io/airbyte/integrations/destination/e2e_test/ThrottledDestinationTest.kt index 929a9d6e111a..367b5909a21c 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/test/kotlin/io/airbyte/integrations/destination/e2e_test/ThrottledDestinationTest.kt +++ b/airbyte-integrations/connectors/destination-e2e-test/src/test/kotlin/io/airbyte/integrations/destination/e2e_test/ThrottledDestinationTest.kt @@ -12,6 +12,7 @@ import java.util.* import java.util.function.Consumer import org.junit.jupiter.api.Test import org.mockito.Mockito +import org.mockito.Mockito.mock /** * This source is designed to be a switch statement for our suite of highly-specific test sources. @@ -30,7 +31,7 @@ class ThrottledDestinationTest { ) ) - val consumer = ThrottledDestination().getConsumer(config, null, outputRecordCollector) + val consumer = ThrottledDestination().getConsumer(config, mock(), outputRecordCollector) consumer.accept(anotherRecord) consumer.accept(anotherRecord) diff --git a/docs/integrations/destinations/e2e-test.md b/docs/integrations/destinations/e2e-test.md index 03985b171a57..37e0294df2cc 100644 --- a/docs/integrations/destinations/e2e-test.md +++ b/docs/integrations/destinations/e2e-test.md @@ -49,7 +49,8 @@ The OSS and Cloud variants have the same version number starting from version `0 | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------- | -| 0.4.1 | 2024-09-18 | [45649](https://github.com/airbytehq/airbyte/pull/45649) | convert test code to kotlin | +| 0.5.0 | 2024-09-18 | [45650](https://github.com/airbytehq/airbyte/pull/45650) | upgrade cdk | +| 0.4.1 | 2024-09-18 | [45649](https://github.com/airbytehq/airbyte/pull/45649) | convert test code to kotlin | | 0.4.0 | 2024-09-18 | [45648](https://github.com/airbytehq/airbyte/pull/45648) | convert production code to kotlin | | 0.3.6 | 2024-05-09 | [38097](https://github.com/airbytehq/airbyte/pull/38097) | Support dedup | | 0.3.5 | 2024-04-29 | [37366](https://github.com/airbytehq/airbyte/pull/37366) | Support refreshes |