Merge pull request #3011 from yufeng-deng/main

sql: adding Google Cloud Spanner sql driver
redpanda-data · Nov 25, 2024 · 015b7f2 · 015b7f2
2 parents 1c54126 + d04c720
commit 015b7f2
Show file tree

Hide file tree

Showing 14 changed files with 1,197 additions and 54 deletions.
diff --git a/docs/modules/components/pages/caches/sql.adoc b/docs/modules/components/pages/caches/sql.adoc
@@ -121,6 +121,7 @@ Options:
 , `snowflake`
 , `trino`
 , `gocosmos`
+, `spanner`
 .
 
 === `dsn`
@@ -129,7 +130,7 @@ A Data Source Name to identify the target database.
 
 ==== Drivers
 
-:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community
+:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community
 
 The following is a list of supported drivers, their placeholder style, and their respective DSN formats:
 
@@ -162,6 +163,9 @@ The following is a list of supported drivers, their placeholder style, and their
 
 | `gocosmos` 
 | https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=<cosmosdb-endpoint>;AccountKey=<cosmosdb-account-key>[;TimeoutMs=<timeout-in-ms>\][;Version=<cosmosdb-api-version>\][;DefaultDb/Db=<db-name>\][;AutoId=<true/false>\][;InsecureSkipVerify=<true/false>\]`^] 
+
+| `spanner` 
+| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] 
 |===
 
 Please note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.

diff --git a/docs/modules/components/pages/inputs/sql_raw.adoc b/docs/modules/components/pages/inputs/sql_raw.adoc
@@ -126,6 +126,7 @@ Options:
 , `snowflake`
 , `trino`
 , `gocosmos`
+, `spanner`
 .
 
 === `dsn`
@@ -134,7 +135,7 @@ A Data Source Name to identify the target database.
 
 ==== Drivers
 
-:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community
+:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community
 
 The following is a list of supported drivers, their placeholder style, and their respective DSN formats:
 
@@ -167,6 +168,9 @@ The following is a list of supported drivers, their placeholder style, and their
 
 | `gocosmos` 
 | https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=<cosmosdb-endpoint>;AccountKey=<cosmosdb-account-key>[;TimeoutMs=<timeout-in-ms>\][;Version=<cosmosdb-api-version>\][;DefaultDb/Db=<db-name>\][;AutoId=<true/false>\][;InsecureSkipVerify=<true/false>\]`^] 
+
+| `spanner` 
+| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] 
 |===
 
 Please note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.

diff --git a/docs/modules/components/pages/inputs/sql_select.adoc b/docs/modules/components/pages/inputs/sql_select.adoc
@@ -134,6 +134,7 @@ Options:
 , `snowflake`
 , `trino`
 , `gocosmos`
+, `spanner`
 .
 
 === `dsn`
@@ -142,7 +143,7 @@ A Data Source Name to identify the target database.
 
 ==== Drivers
 
-:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community
+:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community
 
 The following is a list of supported drivers, their placeholder style, and their respective DSN formats:
 
@@ -175,6 +176,9 @@ The following is a list of supported drivers, their placeholder style, and their
 
 | `gocosmos` 
 | https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=<cosmosdb-endpoint>;AccountKey=<cosmosdb-account-key>[;TimeoutMs=<timeout-in-ms>\][;Version=<cosmosdb-api-version>\][;DefaultDb/Db=<db-name>\][;AutoId=<true/false>\][;InsecureSkipVerify=<true/false>\]`^] 
+
+| `spanner` 
+| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] 
 |===
 
 Please note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.

diff --git a/docs/modules/components/pages/outputs/sql.adoc b/docs/modules/components/pages/outputs/sql.adoc
@@ -106,6 +106,7 @@ Options:
 , `snowflake`
 , `trino`
 , `gocosmos`
+, `spanner`
 .
 
 === `data_source_name`

diff --git a/docs/modules/components/pages/outputs/sql_insert.adoc b/docs/modules/components/pages/outputs/sql_insert.adoc
@@ -143,6 +143,7 @@ Options:
 , `snowflake`
 , `trino`
 , `gocosmos`
+, `spanner`
 .
 
 === `dsn`
@@ -151,7 +152,7 @@ A Data Source Name to identify the target database.
 
 ==== Drivers
 
-:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community
+:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community
 
 The following is a list of supported drivers, their placeholder style, and their respective DSN formats:
 
@@ -184,6 +185,9 @@ The following is a list of supported drivers, their placeholder style, and their
 
 | `gocosmos` 
 | https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=<cosmosdb-endpoint>;AccountKey=<cosmosdb-account-key>[;TimeoutMs=<timeout-in-ms>\][;Version=<cosmosdb-api-version>\][;DefaultDb/Db=<db-name>\][;AutoId=<true/false>\][;InsecureSkipVerify=<true/false>\]`^] 
+
+| `spanner` 
+| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] 
 |===
 
 Please note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.

diff --git a/docs/modules/components/pages/outputs/sql_raw.adoc b/docs/modules/components/pages/outputs/sql_raw.adoc
@@ -138,6 +138,7 @@ Options:
 , `snowflake`
 , `trino`
 , `gocosmos`
+, `spanner`
 .
 
 === `dsn`
@@ -146,7 +147,7 @@ A Data Source Name to identify the target database.
 
 ==== Drivers
 
-:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community
+:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community
 
 The following is a list of supported drivers, their placeholder style, and their respective DSN formats:
 
@@ -179,6 +180,9 @@ The following is a list of supported drivers, their placeholder style, and their
 
 | `gocosmos` 
 | https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=<cosmosdb-endpoint>;AccountKey=<cosmosdb-account-key>[;TimeoutMs=<timeout-in-ms>\][;Version=<cosmosdb-api-version>\][;DefaultDb/Db=<db-name>\][;AutoId=<true/false>\][;InsecureSkipVerify=<true/false>\]`^] 
+
+| `spanner` 
+| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] 
 |===
 
 Please note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.

diff --git a/docs/modules/components/pages/processors/sql.adoc b/docs/modules/components/pages/processors/sql.adoc
@@ -96,6 +96,7 @@ Options:
 , `snowflake`
 , `trino`
 , `gocosmos`
+, `spanner`
 .
 
 === `data_source_name`

diff --git a/docs/modules/components/pages/processors/sql_insert.adoc b/docs/modules/components/pages/processors/sql_insert.adoc
@@ -131,6 +131,7 @@ Options:
 , `snowflake`
 , `trino`
 , `gocosmos`
+, `spanner`
 .
 
 === `dsn`
@@ -139,7 +140,7 @@ A Data Source Name to identify the target database.
 
 ==== Drivers
 
-:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community
+:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community
 
 The following is a list of supported drivers, their placeholder style, and their respective DSN formats:
 
@@ -172,6 +173,9 @@ The following is a list of supported drivers, their placeholder style, and their
 
 | `gocosmos` 
 | https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=<cosmosdb-endpoint>;AccountKey=<cosmosdb-account-key>[;TimeoutMs=<timeout-in-ms>\][;Version=<cosmosdb-api-version>\][;DefaultDb/Db=<db-name>\][;AutoId=<true/false>\][;InsecureSkipVerify=<true/false>\]`^] 
+
+| `spanner` 
+| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] 
 |===
 
 Please note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.

diff --git a/docs/modules/components/pages/processors/sql_raw.adoc b/docs/modules/components/pages/processors/sql_raw.adoc
@@ -143,6 +143,7 @@ Options:
 , `snowflake`
 , `trino`
 , `gocosmos`
+, `spanner`
 .
 
 === `dsn`
@@ -151,7 +152,7 @@ A Data Source Name to identify the target database.
 
 ==== Drivers
 
-:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community
+:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community
 
 The following is a list of supported drivers, their placeholder style, and their respective DSN formats:
 
@@ -184,6 +185,9 @@ The following is a list of supported drivers, their placeholder style, and their
 
 | `gocosmos` 
 | https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=<cosmosdb-endpoint>;AccountKey=<cosmosdb-account-key>[;TimeoutMs=<timeout-in-ms>\][;Version=<cosmosdb-api-version>\][;DefaultDb/Db=<db-name>\][;AutoId=<true/false>\][;InsecureSkipVerify=<true/false>\]`^] 
+
+| `spanner` 
+| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] 
 |===
 
 Please note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.

diff --git a/docs/modules/components/pages/processors/sql_select.adoc b/docs/modules/components/pages/processors/sql_select.adoc
@@ -134,6 +134,7 @@ Options:
 , `snowflake`
 , `trino`
 , `gocosmos`
+, `spanner`
 .
 
 === `dsn`
@@ -142,7 +143,7 @@ A Data Source Name to identify the target database.
 
 ==== Drivers
 
-:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community
+:driver-support: mysql=certified, postgres=certified, clickhouse=community, mssql=community, sqlite=certified, oracle=certified, snowflake=community, trino=community, gocosmos=community, spanner=community
 
 The following is a list of supported drivers, their placeholder style, and their respective DSN formats:
 
@@ -175,6 +176,9 @@ The following is a list of supported drivers, their placeholder style, and their
 
 | `gocosmos` 
 | https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage[`AccountEndpoint=<cosmosdb-endpoint>;AccountKey=<cosmosdb-account-key>[;TimeoutMs=<timeout-in-ms>\][;Version=<cosmosdb-api-version>\][;DefaultDb/Db=<db-name>\][;AutoId=<true/false>\][;InsecureSkipVerify=<true/false>\]`^] 
+
+| `spanner` 
+| projects/[PROJECT]/instances/[INSTANCE]/databases/[DATABASE] 
 |===
 
 Please note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.

diff --git a/go.mod b/go.mod
@@ -4,8 +4,8 @@ replace github.com/99designs/keyring => github.com/Jeffail/keyring v1.2.3
 
 require (
 	cloud.google.com/go/aiplatform v1.68.0
-	cloud.google.com/go/bigquery v1.63.1
-	cloud.google.com/go/pubsub v1.44.0
+	cloud.google.com/go/bigquery v1.64.0
+	cloud.google.com/go/pubsub v1.45.1
 	cloud.google.com/go/storage v1.43.0
 	cloud.google.com/go/vertexai v0.12.0
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
@@ -69,6 +69,7 @@ require (
 	github.com/gofrs/uuid v4.4.0+incompatible
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang-jwt/jwt/v5 v5.2.1
+	github.com/googleapis/go-sql-spanner v1.8.0
 	github.com/gosimple/slug v1.14.0
 	github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c
 	github.com/jackc/pgx/v4 v4.18.3
@@ -145,19 +146,27 @@ require (
 	golang.org/x/net v0.30.0
 	golang.org/x/sync v0.8.0
 	golang.org/x/text v0.19.0
-	google.golang.org/api v0.203.0
+	google.golang.org/api v0.205.0
 	google.golang.org/protobuf v1.35.1
 	modernc.org/sqlite v1.32.0
 )
 
 require (
-	cloud.google.com/go/longrunning v0.6.1 // indirect
+	cel.dev/expr v0.16.1 // indirect
+	cloud.google.com/go/longrunning v0.6.2 // indirect
+	cloud.google.com/go/monitoring v1.21.2 // indirect
 	cloud.google.com/go/secretmanager v1.14.2 // indirect
+	cloud.google.com/go/spanner v1.73.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
+	github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.3 // indirect
+	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
+	github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect
 	github.com/containerd/platforms v0.2.1 // indirect
+	github.com/envoyproxy/go-control-plane v0.13.0 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
 	github.com/hamba/avro/v2 v2.22.2-0.20240625062549-66aad10411d9 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -171,15 +180,17 @@ require (
 	github.com/tidwall/gjson v1.18.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.29.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.29.0 // indirect
 )
 
 require (
 	cloud.google.com/go v0.116.0 // indirect
-	cloud.google.com/go/auth v0.9.9 // indirect
-	cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
+	cloud.google.com/go/auth v0.10.1 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.5 // indirect
 	cloud.google.com/go/compute/metadata v0.5.2 // indirect
-	cloud.google.com/go/iam v1.2.1 // indirect
-	cloud.google.com/go/trace v1.11.1 // indirect
+	cloud.google.com/go/iam v1.2.2 // indirect
+	cloud.google.com/go/trace v1.11.2 // indirect
 	cuelang.org/go v0.9.2 // indirect
 	dario.cat/mergo v1.0.0 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
@@ -401,10 +412,10 @@ require (
 	golang.org/x/time v0.7.0 // indirect
 	golang.org/x/tools v0.26.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
-	google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect
-	google.golang.org/grpc v1.67.1
+	google.golang.org/genproto v0.0.0-20241113202542-65e8d215514f // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241113202542-65e8d215514f // indirect
+	google.golang.org/grpc v1.68.0
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
 	gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect