Skip to content

Commit e34af58

Browse files
xinlian12annie-mac
xinlian12
and
annie-mac
authored
KafakV2SourceConnector (Azure#39410)
* kafkaV2SourceConnector --------- Co-authored-by: annie-mac <[email protected]>
1 parent bf1dad9 commit e34af58

File tree

64 files changed

+6081
-39
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+6081
-39
lines changed

‎.vscode/cspell.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@
140140
"sdk/cosmos/azure-cosmos-encryption/**",
141141
"sdk/cosmos/azure-cosmos-spark_3_2-12/**",
142142
"sdk/spring/azure-spring-data-cosmos/**",
143+
"sdk/cosmos/azure-cosmos-kafka-connect/**",
143144
"sdk/deviceupdate/azure-iot-deviceupdate/**",
144145
"sdk/e2e/src/**",
145146
"sdk/eventgrid/azure-messaging-eventgrid-cloudnative-cloudevents/**",
@@ -723,7 +724,7 @@
723724
"words": [
724725
"Pfast",
725726
"Pdirect",
726-
"Pmulti",
727+
"Pmulti",
727728
"Psplit",
728729
"Pquery",
729730
"Pcfp",

‎eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,9 @@ the main ServiceBusClientBuilder. -->
316316
files="com.azure.cosmos.ClientSideRequestStatistics"/> <!-- Need OperatingSystemMXBean from sun to obtain cpu info -->
317317
<suppress checks="EnforceFinalFields" files="com.azure.spring.cloud.config.AppConfigurationPropertySourceLocator"/>
318318
<suppress checks="ConstantName" files="com.azure.spring.cloud.config.AppConfigurationPropertySourceLocator"/>
319+
<suppress checks="com.azure.tools.checkstyle.checks.GoodLoggingCheck"
320+
files="[/\\]azure-cosmos-kafka-connect[/\\]"/>
321+
<suppress checks="com.azure.tools.checkstyle.checks.ExternalDependencyExposedCheck" files="com.azure.cosmos.kafka.connect.CosmosDBSourceConnector"/>
319322

320323
<!-- Checkstyle suppressions for resource manager package -->
321324
<suppress checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" files="com.azure.resourcemanager.*"/>

‎eng/versioning/external_dependencies.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -395,6 +395,10 @@ cosmos_org.scalastyle:scalastyle-maven-plugin;1.0.0
395395
# Cosmos Kafka connector runtime dependencies
396396
cosmos_org.apache.kafka:connect-api;3.6.0
397397
# Cosmos Kafka connector tests only
398+
cosmos_org.apache.kafka:connect-runtime;3.6.0
399+
cosmos_org.testcontainers:testcontainers;1.19.5
400+
cosmos_org.testcontainers:kafka;1.19.5
401+
cosmos_org.sourcelab:kafka-connect-client;4.0.4
398402
# Maven Tools for Cosmos Kafka connector only
399403
cosmos_io.confluent:kafka-connect-maven-plugin;0.12.0
400404

‎sdk/cosmos/azure-cosmos-kafka-connect/CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
### 1.0.0-beta.1 (Unreleased)
44

55
#### Features Added
6+
* Added Source connector. See [PR 39410](https://github.com/Azure/azure-sdk-for-java/pull/39410)
67

78
#### Breaking Changes
89

‎sdk/cosmos/azure-cosmos-kafka-connect/doc/configuration-reference.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
## Configuration Reference:
2+
3+
## Generic Configuration
4+
| Config Property Name | Default | Description |
5+
|:---------------------------------------------|:--------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
6+
| `kafka.connect.cosmos.accountEndpoint` | None | Cosmos DB Account Endpoint Uri |
7+
| `kafka.connect.cosmos.accountEndpoint` | None | Cosmos DB Account Key |
8+
| `kafka.connect.cosmos.useGatewayMode` | `false` | Flag to indicate whether to use gateway mode. By default it is false. |
9+
| `kafka.connect.cosmos.preferredRegionsList` | `[]` | Preferred regions list to be used for a multi region Cosmos DB account. This is a comma separated value (e.g., `[East US, West US]` or `East US, West US`) provided preferred regions will be used as hint. You should use a collocated kafka cluster with your Cosmos DB account and pass the kafka cluster region as preferred region. See list of azure regions [here](https://docs.microsoft.com/dotnet/api/microsoft.azure.documents.locationnames?view=azure-dotnet&preserve-view=true). |
10+
| `kafka.connect.cosmos.applicationName` | `""` | Application name. Will be added as the userAgent suffix. |
11+
12+
## Source Connector Configuration
13+
| Config Property Name | Default | Description |
14+
|:----------------------------------------------------------|:-------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
15+
| `kafka.connect.cosmos.source.database.name` | None | Cosmos DB database name. |
16+
| `kafka.connect.cosmos.source.containers.includeAll` | `false` | Flag to indicate whether reading from all containers. |
17+
| `kafka.connect.cosmos.source.containers.includedList` | `[]` | Containers included. This config will be ignored if kafka.connect.cosmos.source.includeAllContainers is true. |
18+
| `kafka.connect.cosmos.source.containers.topicMap` | `[]` | A comma delimited list of Kafka topics mapped to Cosmos containers. For example: topic1#con1,topic2#con2. By default, container name is used as the name of the kafka topic to publish data to, can use this property to override the default config |
19+
| `kafka.connect.cosmos.source.changeFeed.startFrom` | `Beginning` | ChangeFeed Start from settings (Now, Beginning or a certain point in time (UTC) for example 2020-02-10T14:15:03) - the default value is 'Beginning'. |
20+
| `kafka.connect.cosmos.source.changeFeed.mode` | `LatestVersion` | ChangeFeed mode (LatestVersion or AllVersionsAndDeletes). |
21+
| `kafka.connect.cosmos.source.changeFeed.maxItemCountHint` | `1000` | The maximum number of documents returned in a single change feed request. But the number of items received might be higher than the specified value if multiple items are changed by the same transaction. |
22+
| `kafka.connect.cosmos.source.metadata.poll.delay.ms` | `300000` | Indicates how often to check the metadata changes (including container split/merge, adding/removing/recreated containers). When changes are detected, it will reconfigure the tasks. Default is 5 minutes. |
23+
| `kafka.connect.cosmos.source.metadata.storage.topic` | `_cosmos.metadata.topic` | The name of the topic where the metadata are stored. The metadata topic will be created if it does not already exist, else it will use the pre-created topic. |
24+
| `kafka.connect.cosmos.source.messageKey.enabled` | `true` | Whether to set the kafka record message key. |
25+
| `kafka.connect.cosmos.source.messageKey.field` | `id` | The field to use as the message key. |

‎sdk/cosmos/azure-cosmos-kafka-connect/pom.xml‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,6 @@ Licensed under the MIT License.
3737
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
3838
<jacoco.min.linecoverage>0.01</jacoco.min.linecoverage>
3939
<jacoco.min.branchcoverage>0.02</jacoco.min.branchcoverage>
40-
<maven.compiler.source>11</maven.compiler.source>
41-
<maven.compiler.target>11</maven.compiler.target>
4240
<shadingPrefix>azure_cosmos_kafka_connect</shadingPrefix>
4341

4442
<!-- CosmosSkip - This is not a module we want/expect external customers to consume. Skip breaking API checks. -->
@@ -48,7 +46,12 @@ Licensed under the MIT License.
4846
<javaModulesSurefireArgLine>
4947
--add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect=ALL-UNNAMED
5048
--add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect.implementation=ALL-UNNAMED
51-
--add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect.models=ALL-UNNAMED
49+
--add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect.implementation.source=com.fasterxml.jackson.databind,ALL-UNNAMED
50+
--add-opens com.azure.cosmos/com.azure.cosmos.implementation.routing=ALL-UNNAMED
51+
--add-opens com.azure.cosmos/com.azure.cosmos.implementation.apachecommons.lang=ALL-UNNAMED
52+
--add-exports com.azure.cosmos/com.azure.cosmos.implementation.changefeed.common=com.azure.cosmos.kafka.connect
53+
--add-exports com.azure.cosmos/com.azure.cosmos.implementation.feedranges=com.azure.cosmos.kafka.connect
54+
--add-exports com.azure.cosmos/com.azure.cosmos.implementation.query=com.azure.cosmos.kafka.connect
5255
</javaModulesSurefireArgLine>
5356
</properties>
5457

@@ -94,6 +97,19 @@ Licensed under the MIT License.
9497
<version>1.10.0</version> <!-- {x-version-update;org.apache.commons:commons-text;external_dependency} -->
9598
</dependency>
9699

100+
<dependency>
101+
<groupId>org.apache.kafka</groupId>
102+
<artifactId>connect-runtime</artifactId>
103+
<version>3.6.0</version> <!-- {x-version-update;cosmos_org.apache.kafka:connect-runtime;external_dependency} -->
104+
<scope>test</scope>
105+
<exclusions>
106+
<exclusion>
107+
<artifactId>jackson-jaxrs-json-provider</artifactId>
108+
<groupId>com.fasterxml.jackson.jaxrs</groupId>
109+
</exclusion>
110+
</exclusions>
111+
</dependency>
112+
97113
<dependency>
98114
<groupId>org.testng</groupId>
99115
<artifactId>testng</artifactId>
@@ -160,6 +176,24 @@ Licensed under the MIT License.
160176
<version>1.14.12</version> <!-- {x-version-update;testdep_net.bytebuddy:byte-buddy-agent;external_dependency} -->
161177
<scope>test</scope>
162178
</dependency>
179+
<dependency>
180+
<groupId>org.testcontainers</groupId>
181+
<artifactId>testcontainers</artifactId>
182+
<version>1.19.5</version> <!-- {x-version-update;cosmos_org.testcontainers:testcontainers;external_dependency} -->
183+
<scope>test</scope>
184+
</dependency>
185+
<dependency>
186+
<groupId>org.testcontainers</groupId>
187+
<artifactId>kafka</artifactId>
188+
<version>1.19.5</version> <!-- {x-version-update;cosmos_org.testcontainers:kafka;external_dependency} -->
189+
<scope>test</scope>
190+
</dependency>
191+
<dependency>
192+
<groupId>org.sourcelab</groupId>
193+
<artifactId>kafka-connect-client</artifactId>
194+
<version>4.0.4</version> <!-- {x-version-update;cosmos_org.sourcelab:kafka-connect-client;external_dependency} -->
195+
<scope>test</scope>
196+
</dependency>
163197
</dependencies>
164198

165199
<build>
@@ -204,6 +238,7 @@ Licensed under the MIT License.
204238
<include>com.azure:*</include>
205239
<include>org.apache.kafka:connect-api:[3.6.0]</include> <!-- {x-include-update;cosmos_org.apache.kafka:connect-api;external_dependency} -->
206240
<include>io.confluent:kafka-connect-maven-plugin:[0.12.0]</include> <!-- {x-include-update;cosmos_io.confluent:kafka-connect-maven-plugin;external_dependency} -->
241+
<include>org.sourcelab:kafka-connect-client:[4.0.4]</include> <!-- {x-include-update;cosmos_org.sourcelab:kafka-connect-client;external_dependency} -->
207242
</includes>
208243
</bannedDependencies>
209244
</rules>
@@ -221,6 +256,7 @@ Licensed under the MIT License.
221256
<goal>shade</goal>
222257
</goals>
223258
<configuration>
259+
<finalName>${project.artifactId}-${project.version}-jar-with-dependencies</finalName>
224260
<filters>
225261
<filter>
226262
<artifact>*:*:*:*</artifact>

‎sdk/cosmos/azure-cosmos-kafka-connect/src/docker/.gitignore‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
connectors/
2+
log.txt
3+
4+
# Exclude all temporary files in resources
5+
!resources/*example
6+
resources/sink.properties
7+
resources/source.properties
8+
resources/standalone.properties

‎sdk/cosmos/azure-cosmos-kafka-connect/src/docker/Dockerfile‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Build the Cosmos DB Connectors on top of the Kafka Connect image
2+
FROM confluentinc/cp-kafka-connect:7.5.0
3+
4+
# Install datagen connector
5+
RUN confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:latest
6+
7+
COPY connectors/ /etc/kafka-connect/jars

0 commit comments

Comments
 (0)