Skip to content

Commit c082252

Browse files
johndee7310johndee7310
committed
Add Managed Kafka ConnectCluster and Connector resources and tests
1 parent f181a08 commit c082252

File tree

4 files changed

+393
-0
lines changed

4 files changed

+393
-0
lines changed

‎mmv1/products/managedkafka/ConnectCluster.yaml

+147
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
# Copyright 2025 Google Inc.
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
14+
---
15+
name: 'ConnectCluster'
16+
description: A Managed Service for Kafka Connect cluster.
17+
min_version: 'beta'
18+
docs:
19+
id_format: 'projects/{{project}}/locations/{{location}}/connectClusters/{{connect_cluster_id}}'
20+
base_url: 'projects/{{project}}/locations/{{location}}/connectClusters'
21+
self_link: 'projects/{{project}}/locations/{{location}}/connectClusters/{{connect_cluster_id}}'
22+
create_url: 'projects/{{project}}/locations/{{location}}/connectClusters?connectClusterId={{connect_cluster_id}}'
23+
update_verb: 'PATCH'
24+
update_mask: true
25+
import_format:
26+
- 'projects/{{project}}/locations/{{location}}/connectClusters/{{connect_cluster_id}}'
27+
timeouts:
28+
insert_minutes: 60
29+
update_minutes: 30
30+
delete_minutes: 30
31+
autogen_async: true
32+
async:
33+
actions: ['create', 'delete', 'update']
34+
type: 'OpAsync'
35+
operation:
36+
base_url: '{{op_id}}'
37+
timeouts:
38+
result:
39+
resource_inside_response: true
40+
custom_code:
41+
examples:
42+
- name: 'managedkafka_connect_cluster_basic'
43+
primary_resource_id: 'example'
44+
min_version: 'beta'
45+
vars:
46+
connect_cluster_id: 'my-connect-cluster'
47+
cluster_id: 'my-cluster'
48+
subnetwork_id: 'my-subnetwork'
49+
network_id: 'my-network'
50+
parameters:
51+
- name: 'location'
52+
type: String
53+
description: "ID of the location of the Kafka Connect resource. See
54+
https://cloud.google.com/managed-kafka/docs/locations for a list of
55+
supported locations."
56+
required: true
57+
immutable: true
58+
url_param_only: true
59+
- name: 'connectClusterId'
60+
type: String
61+
description: "The ID to use for the Connect Cluster, which will become the final
62+
component of the connect cluster's name. This value is structured like: `my-connect-cluster-id`."
63+
url_param_only: true
64+
required: true
65+
immutable: true
66+
properties:
67+
- name: 'name'
68+
type: String
69+
description: "The name of the connect cluster. Structured like: `projects/PROJECT_ID/locations/LOCATION/connectClusters/CONNECT_CLUSTER_ID`."
70+
output: true
71+
- name: 'kafkaCluster'
72+
type: String
73+
description: "The name of the Kafka cluster this Kafka Connect cluster is attached to. Structured like: `projects/PROJECT_ID/locations/LOCATION/clusters/CLUSTER_ID`."
74+
required: true
75+
- name: 'createTime'
76+
type: String
77+
description: "The time when the cluster was created."
78+
output: true
79+
- name: 'updateTime'
80+
type: String
81+
description: "The time when the cluster was last updated."
82+
output: true
83+
- name: 'labels'
84+
type: KeyValueLabels
85+
description: "List of label KEY=VALUE pairs to add. Keys must start with a lowercase
86+
character and contain only hyphens (-), underscores (\_), lowercase
87+
characters, and numbers. Values must contain only hyphens (-),
88+
underscores (\_), lowercase characters, and numbers."
89+
- name: 'capacityConfig'
90+
type: NestedObject
91+
description: "A capacity configuration of a Kafka cluster."
92+
required: true
93+
properties:
94+
- name: vcpuCount
95+
type: String
96+
description: "The number of vCPUs to provision for the cluster. The minimum is 3."
97+
required: true
98+
- name: memoryBytes
99+
type: String
100+
description: "The memory to provision for the cluster in bytes. The CPU:memory ratio (vCPU:GiB) must be between 1:1 and 1:8. Minimum: 3221225472 (3 GiB)."
101+
required: true
102+
- name: 'gcpConfig'
103+
type: NestedObject
104+
description: "Configuration properties for a Kafka Connect cluster deployed to Google Cloud Platform."
105+
required: true
106+
properties:
107+
- name: 'accessConfig'
108+
type: NestedObject
109+
description: "The configuration of access to the Kafka Connect cluster."
110+
required: true
111+
properties:
112+
- name: 'networkConfigs'
113+
type: Array
114+
description: "Virtual Private Cloud (VPC) subnets where IP addresses for the Kafka Connect
115+
cluster are allocated. To make the connect cluster available in a VPC, you must specify at least
116+
one subnet per network. You must specify between 1 and 10 subnets.
117+
Additional subnets may be specified with additional `network_configs` blocks."
118+
required: true
119+
item_type:
120+
type: NestedObject
121+
properties:
122+
- name: 'primarySubnet'
123+
type: String
124+
description: "VPC subnet to make available to the Kafka Connect cluster. Structured like: projects/{project}/regions/{region}/subnetworks/{subnet_id}.
125+
It is used to create a Private Service Connect (PSC) interface for the Kafka Connect workers. It must be located in the same region as the
126+
Kafka Connect cluster. The CIDR range of the subnet must be within the IPv4 address ranges for private networks, as specified in RFC 1918.
127+
The primary subnet CIDR range must have a minimum size of /22 (1024 addresses)."
128+
required: true
129+
diff_suppress_func: 'tpgresource.ProjectNumberDiffSuppress'
130+
- name: 'additionalSubnets'
131+
type: Array
132+
description: "Additional subnets may be specified. They may be in another region, but must be in the
133+
same VPC network. The Connect workers can communicate with
134+
network endpoints in either the primary or additional subnets."
135+
item_type:
136+
type: String
137+
- name: 'dnsDomainNames'
138+
type: Array
139+
description: "Additional DNS domain names from the subnet's network to be made visible to the Connect Cluster. When using
140+
MirrorMaker2, it's necessary to add the bootstrap address's dns domain name of the target cluster to make it visible to
141+
the connector. For example: my-kafka-cluster.us-central1.managedkafka.my-project.cloud.goog"
142+
item_type:
143+
type: String
144+
- name: 'state'
145+
type: String
146+
description: "The current state of the connect cluster. Possible values: `STATE_UNSPECIFIED`, `CREATING`, `ACTIVE`, `DELETING`."
147+
output: true

‎mmv1/products/managedkafka/Connector.yaml

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# Copyright 2025 Google Inc.
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
14+
---
15+
name: 'Connector'
16+
description: A Managed Service for Kafka Connect Connectors.
17+
min_version: 'beta'
18+
docs:
19+
id_format: 'projects/{{project}}/locations/{{location}}/connectClusters/{{connect_cluster}}/connectors/{{connector_id}}'
20+
base_url: 'projects/{{project}}/locations/{{location}}/connectClusters/{{connect_cluster}}/connectors'
21+
self_link: 'projects/{{project}}/locations/{{location}}/connectClusters/{{connect_cluster}}/connectors/{{connector_id}}'
22+
create_url: 'projects/{{project}}/locations/{{location}}/connectClusters/{{connect_cluster}}/connectors?connectorId={{connector_id}}'
23+
update_verb: 'PATCH'
24+
update_mask: true
25+
import_format:
26+
- 'projects/{{project}}/locations/{{location}}/connectClusters/{{connect_cluster}}/connectors/{{connector_id}}'
27+
timeouts:
28+
insert_minutes: 60
29+
update_minutes: 30
30+
delete_minutes: 30
31+
examples:
32+
- name: 'managedkafka_connector_basic'
33+
primary_resource_id: 'example'
34+
min_version: 'beta'
35+
vars:
36+
connector_id: 'my-connector'
37+
connect_cluster_id: 'my-connect-cluster'
38+
cluster_id: 'my-cluster'
39+
subnetwork_id: 'my-subnetwork'
40+
network_id: 'my-network'
41+
parameters:
42+
- name: 'location'
43+
type: String
44+
description: "ID of the location of the Kafka Connect resource. See
45+
https://cloud.google.com/managed-kafka/docs/locations for a list of
46+
supported locations."
47+
url_param_only: true
48+
required: true
49+
immutable: true
50+
- name: 'connectCluster'
51+
type: String
52+
description: "The connect cluster name."
53+
url_param_only: true
54+
required: true
55+
immutable: true
56+
- name: 'connectorId'
57+
type: String
58+
description: "The ID to use for the connector, which will become the final
59+
component of the connector's name. This value is structured like: `my-connector-id`."
60+
url_param_only: true
61+
required: true
62+
immutable: true
63+
properties:
64+
- name: 'name'
65+
type: String
66+
description: "The name of the connector. The `connector` segment is used when
67+
connecting directly to the connect cluster. Structured like: `projects/PROJECT_ID/locations/LOCATION/connectClusters/CONNECT_CLUSTER/connectors/CONNECTOR_ID`."
68+
output: true
69+
- name: 'configs'
70+
type: KeyValuePairs
71+
description: "Connector config as keys/values. The keys of the map are connector property names, for example:
72+
`connector.class`, `tasks.max`, `key.converter`."
73+
- name: 'state'
74+
type: String
75+
description: "The current state of the connect. Possible values: `STATE_UNSPECIFIED`, `UNASSIGNED`, `RUNNING`, `PAUSED`, `FAILED`, `RESTARTING`, and `STOPPED`."
76+
output: true
77+
- name: 'taskRestartPolicy'
78+
type: NestedObject
79+
description: "A policy that specifies how to restart the failed connectors/tasks in a Cluster resource. If not set, the failed connectors/tasks won't be restarted."
80+
properties:
81+
- name: 'minimumBackoff'
82+
type: String
83+
description: |
84+
The minimum amount of time to wait before retrying a failed task. This sets a lower bound for the backoff delay.
85+
A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s".
86+
- name: 'maximumBackoff'
87+
type: String
88+
description: |
89+
The maximum amount of time to wait before retrying a failed task. This sets an upper bound for the backoff delay.
90+
A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s".

‎mmv1/templates/terraform/examples/managedkafka_connect_cluster_basic.tf.tmpl

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
provider "google-beta" {
2+
managed_kafka_custom_endpoint = "https://staging-managedkafka.sandbox.googleapis.com/v1/"
3+
}
4+
5+
resource "google_compute_network" "new_network" {
6+
name = "{{index $.Vars "network_id"}}"
7+
auto_create_subnetworks = false
8+
9+
provider = google-beta
10+
}
11+
12+
resource "google_compute_subnetwork" "new_subnet" {
13+
name = "{{index $.Vars "subnetwork_id"}}"
14+
ip_cidr_range = "10.2.0.0/16"
15+
region = "us-central1"
16+
network = google_compute_network.new_network.id
17+
18+
provider = google-beta
19+
}
20+
21+
resource "google_managed_kafka_cluster" "gmk_cluster" {
22+
cluster_id = "{{index $.Vars "cluster_id"}}"
23+
location = "us-central1"
24+
capacity_config {
25+
vcpu_count = 3
26+
memory_bytes = 3221225472
27+
}
28+
gcp_config {
29+
access_config {
30+
network_configs {
31+
subnet = "projects/${data.google_project.project.project_id}/regions/us-central1/subnetworks/default"
32+
}
33+
}
34+
}
35+
36+
provider = google-beta
37+
}
38+
39+
resource "google_managed_kafka_connect_cluster" "{{$.PrimaryResourceId}}" {
40+
connect_cluster_id = "{{index $.Vars "connect_cluster_id"}}"
41+
kafka_cluster = "projects/${data.google_project.project.project_id}/locations/us-central1/clusters/${google_managed_kafka_cluster.gmk_cluster.cluster_id}"
42+
location = "us-central1"
43+
capacity_config {
44+
vcpu_count = 12
45+
memory_bytes = 21474836480
46+
}
47+
gcp_config {
48+
access_config {
49+
network_configs {
50+
primary_subnet = "projects/${data.google_project.project.project_id}/regions/us-central1/subnetworks/default"
51+
additional_subnets = ["${google_compute_subnetwork.new_subnet.id}"]
52+
dns_domain_names = ["${google_managed_kafka_cluster.gmk_cluster.cluster_id}.us-central1.managedkafka-staging.${data.google_project.project.project_id}.cloud-staging.goog"]
53+
}
54+
}
55+
}
56+
labels = {
57+
key = "value"
58+
}
59+
60+
depends_on = [google_managed_kafka_cluster.gmk_cluster]
61+
62+
provider = google-beta
63+
}
64+
65+
data "google_project" "project" {
66+
provider = google-beta
67+
}

‎mmv1/templates/terraform/examples/managedkafka_connector_basic.tf.tmpl

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
provider "google-beta" {
2+
managed_kafka_custom_endpoint = "https://staging-managedkafka.sandbox.googleapis.com/v1/"
3+
}
4+
5+
resource "google_compute_network" "new_network" {
6+
name = "{{index $.Vars "network_id"}}"
7+
auto_create_subnetworks = false
8+
9+
provider = google-beta
10+
}
11+
12+
resource "google_compute_subnetwork" "new_subnet" {
13+
name = "{{index $.Vars "subnetwork_id"}}"
14+
ip_cidr_range = "10.2.0.0/16"
15+
region = "us-central1"
16+
network = google_compute_network.new_network.id
17+
18+
provider = google-beta
19+
}
20+
21+
resource "google_managed_kafka_cluster" "gmk_cluster" {
22+
cluster_id = "{{index $.Vars "cluster_id"}}"
23+
location = "us-central1"
24+
capacity_config {
25+
vcpu_count = 3
26+
memory_bytes = 3221225472
27+
}
28+
gcp_config {
29+
access_config {
30+
network_configs {
31+
subnet = "projects/${data.google_project.project.project_id}/regions/us-central1/subnetworks/default"
32+
}
33+
}
34+
}
35+
36+
provider = google-beta
37+
}
38+
39+
resource "google_managed_kafka_connect_cluster" "mkc_cluster" {
40+
connect_cluster_id = "{{index $.Vars "connect_cluster_id"}}"
41+
kafka_cluster = "projects/${data.google_project.project.project_id}/locations/us-central1/clusters/${google_managed_kafka_cluster.gmk_cluster.cluster_id}"
42+
location = "us-central1"
43+
capacity_config {
44+
vcpu_count = 12
45+
memory_bytes = 21474836480
46+
}
47+
gcp_config {
48+
access_config {
49+
network_configs {
50+
primary_subnet = "projects/${data.google_project.project.project_id}/regions/us-central1/subnetworks/default"
51+
additional_subnets = ["${google_compute_subnetwork.new_subnet.id}"]
52+
dns_domain_names = ["${google_managed_kafka_cluster.gmk_cluster.cluster_id}.us-central1.managedkafka-staging.${data.google_project.project.project_id}.cloud-staging.goog"]
53+
}
54+
}
55+
}
56+
labels = {
57+
key = "value"
58+
}
59+
60+
depends_on = [google_managed_kafka_cluster.gmk_cluster]
61+
62+
provider = google-beta
63+
}
64+
65+
resource "google_managed_kafka_connector" "{{$.PrimaryResourceId}}" {
66+
connector_id = "{{index $.Vars "connector_id"}}"
67+
connect_cluster = google_managed_kafka_connect_cluster.mkc_cluster.connect_cluster_id
68+
location = "us-central1"
69+
configs = {
70+
"connector.class" = "com.google.pubsub.kafka.sink.CloudPubSubSinkConnector"
71+
"tasks.max" = "1"
72+
"topics" = "gmk-topic"
73+
"name" = "{{index $.Vars "connector_id"}}"
74+
"value.converter" = "org.apache.kafka.connect.storage.StringConverter"
75+
"key.converter" = "org.apache.kafka.connect.storage.StringConverter"
76+
"cps.topic" = "gmk-cps-sink-topic",
77+
"cps.project" = "${data.google_project.project.project_id}"
78+
}
79+
task_restart_policy {
80+
minimum_backoff = "60s"
81+
maximum_backoff = "1800s"
82+
}
83+
84+
provider = google-beta
85+
}
86+
87+
data "google_project" "project" {
88+
provider = google-beta
89+
}

0 commit comments

Comments
 (0)