Merge branch 'master' into master

.github/workflows/check-codegen.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        go-version: [1.20.x, 1.21.x]
+        go-version: [1.21.x, 1.22.x]
     runs-on: ${{ matrix.os }}
     steps:
     - name: Checkout
@@ -32,7 +32,7 @@ jobs:
     - name: Build
       run: |
         go install ./internal/cmd/gtrace
-        go install go.uber.org/mock/mockgen@892b665398ecece7c7a900d2a2f2fa3dac07e4c4        
+        go install go.uber.org/mock/mockgen@v0.4.0
 
     - name: Clean and re-generate *_gtrace.go files
       run: |

.github/workflows/examples.yml

@@ -15,11 +15,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ydb-version: [ 22.5, 23.1, 23.2 ]
-        application: [ native, database_sql, gorm, xorm ]
+        ydb-version: [ 23.3, 24.1 ]
+        application: [ native/table, native/query, database_sql, gorm, xorm ]
     services:
       ydb:
-        image: cr.yandex/crpsjg1coh47p81vh2lc/yandex-docker-local-ydb:${{ matrix.ydb-version }}
+        image: ydbplatform/local-ydb:${{ matrix.ydb-version }}
         ports:
           - 2135:2135
           - 2136:2136
@@ -29,11 +29,13 @@ jobs:
         env:
           YDB_LOCAL_SURVIVE_RESTART: true
           YDB_USE_IN_MEMORY_PDISKS: true
+          YDB_TABLE_ENABLE_PREPARED_DDL: true
         options: '-h localhost'
     env:
       OS: ubuntu-latest
       YDB_CONNECTION_STRING: grpc://localhost:2136/local
       YDB_ANONYMOUS_CREDENTIALS: 1
+      YDB_VERSION: ${{ matrix.ydb-version }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v3

.github/workflows/publish.yml

@@ -70,7 +70,7 @@ jobs:
             TAG="v$MAJOR.$MINOR.$PATCH";
           fi;
           git tag $TAG
-          git push --tags && git push
+          git push && git push --tags
           CHANGELOG="$CHANGELOG
           
           Full Changelog: [$LAST_TAG...$TAG](https://github.com/ydb-platform/ydb-go-sdk/compare/$LAST_TAG...$TAG)"

.github/workflows/slo.yml

@@ -39,33 +39,39 @@ jobs:
           s3_images_folder: ${{ vars.SLO_S3_IMAGES_FOLDER }}
           grafana_domain: ${{ vars.SLO_GRAFANA_DOMAIN }}
           grafana_dashboard: ${{ vars.SLO_GRAFANA_DASHBOARD }}
-          ydb_version: 'newest'
+          ydb_version: '24.1.7'
           timeBetweenPhases: 30
           shutdownTime: 30
 
-          language_id0: 'native'
+          language_id0: 'native-table'
           workload_path0: 'tests/slo'
-          language0: 'Go SDK native'
+          language0: 'Native ydb-go-sdk/v3 over table-service'
           workload_build_context0: ../..
-          workload_build_options0: -f Dockerfile --build-arg SRC_PATH=native --build-arg JOB_NAME=workload-native
+          workload_build_options0: -f Dockerfile --build-arg SRC_PATH=native/table --build-arg JOB_NAME=workload-native-table
 
-          language_id1: 'databasesql'
+          language_id1: 'native-query'
           workload_path1: 'tests/slo'
-          language1: 'Go SDK database/sql'
+          language1: 'Native ydb-go-sdk/v3 over query-service'
           workload_build_context1: ../..
-          workload_build_options1: -f Dockerfile --build-arg SRC_PATH=database/sql --build-arg JOB_NAME=workload-databasesql
+          workload_build_options1: -f Dockerfile --build-arg SRC_PATH=native/query --build-arg JOB_NAME=workload-native-query
 
-          language_id2: 'gorm'
+          language_id2: 'database-sql'
           workload_path2: 'tests/slo'
-          language2: 'Go SDK gorm'
+          language2: 'Go SDK database/sql'
           workload_build_context2: ../..
-          workload_build_options2: -f Dockerfile --build-arg SRC_PATH=gorm --build-arg JOB_NAME=workload-gorm
+          workload_build_options2: -f Dockerfile --build-arg SRC_PATH=database/sql --build-arg JOB_NAME=workload-database-sql
 
-          language_id3: 'xorm'
+          language_id3: 'gorm'
           workload_path3: 'tests/slo'
-          language3: 'Go SDK xorm'
+          language3: 'Go SDK gorm'
           workload_build_context3: ../..
-          workload_build_options3: -f Dockerfile --build-arg SRC_PATH=xorm --build-arg JOB_NAME=workload-xorm
+          workload_build_options3: -f Dockerfile --build-arg SRC_PATH=gorm --build-arg JOB_NAME=workload-gorm
+
+          language_id4: 'xorm'
+          workload_path4: 'tests/slo'
+          language4: 'Go SDK xorm'
+          workload_build_context4: ../..
+          workload_build_options4: -f Dockerfile --build-arg SRC_PATH=xorm --build-arg JOB_NAME=workload-xorm
 
       - uses: actions/upload-artifact@v3
         if: always()

.github/workflows/tests.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go-version: [1.20.x, 1.21.x]
+        go-version: [1.21.x, 1.22.x]
         os: [ubuntu, windows, macOS]
     env:
       OS: ${{ matrix.os }}-latest
@@ -44,11 +44,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go-version: [1.20.x, 1.21.x]
-        ydb-version: [22.5, 23.1, 23.2, 23.3]
+        go-version: [1.21.x, 1.22.x]
+        ydb-version: [23.3, 24.1]
     services:
       ydb:
-        image: cr.yandex/yc/yandex-docker-local-ydb:${{ matrix.ydb-version }}
+        image: ydbplatform/local-ydb:${{ matrix.ydb-version }}
         ports:
           - 2135:2135
           - 2136:2136
@@ -58,6 +58,7 @@ jobs:
         env:
           YDB_LOCAL_SURVIVE_RESTART: true
           YDB_USE_IN_MEMORY_PDISKS: true
+          YDB_TABLE_ENABLE_PREPARED_DDL: true
         options: '-h localhost'
     env:
       OS: ubuntu-latest

.golangci.yml

@@ -112,6 +112,10 @@ linters-settings:
     # if it's called for subdir of a project it can't find external interfaces. All text editor integrations
     # with golangci-lint call it on a directory with the changed file.
     check-exported: false
+  gomoddirectives:
+    replace-local: true
+    replace-allow-list:
+      - xorm.io/xorm
   gocritic:
     disabled-checks:
       - whyNoLint # https://github.com/go-critic/go-critic/issues/1063
@@ -219,7 +223,6 @@ linters:
     - cyclop
     - depguard
     - dupl
-    - errname
     - exhaustive
     - exhaustivestruct
     - exhaustruct
@@ -231,15 +234,12 @@ linters:
     - goerr113
     - golint
     - gomnd
-    - gomoddirectives
     - ifshort
     - interfacebloat
     - ireturn
     - maintidx
-    - maligned
     - nonamedreturns
     - paralleltest
-    - protogetter
     - scopelint
     - structcheck
     - testableexamples

CHANGELOG.md

@@ -1,28 +1,32 @@
-* Refactored `log/table.go` and extract funcs for gocognit linter
-* Refactored `internal/cmd/gtrace/main.go` and extract funcs for gocognit linter
-* Refactored `internal/bind/params.go` and extract func `params.checkDriverNamedValue` for gocognit linter
-* Refactored `internal/cmd/gtrace/main.go` and extract funcs for gocognit linter
-* Refactored `retry/retry.go` and extract func `retry.RecoveryCallbackWrapper` for gocognit linter
-* Refactored `retry/sql_test.go` and extract func `sql_test.canRetry` for gocognit linter
-* Refactored `internal/topic/topicwriterinternal/queue_test.go` and extract funcs for gocognit linter
-* Refactored `log/driver.go` and extract funcs for gocognit linter
-* Refactored `internal/table/scanner/scanner.go` and extract funcs for gocognit linter
-* Refactored `log/topic.go` and extract funcs for gocognit linter
-* Refactored `internal/table/scanner/scanner_test.go` and extract funcs for gocognit linter
-* Refactored `log/sql.go` and extract funcs for gocognit linter
-* Refactored `metrics/driver.go` and extract funcs for gocognit linter
-* Refactored `metrics/sql.go` and extract funcs for gocognit linter
-* Refactored `internal/table/retry_test.go` and extract func `retry_test.checkResultsRetryWithCustomErrors` for gocognit linter
-* Refactored `internal/table/client.go` and extract func `client.onCloseSession` for gocognit linter
-* Refactored `internal/decimal/decimal.go` and extract func `decimal.dotStringAnalysis` for gocognit linter
-* Refactored `internal/backoff/backoff_test.go` and extract func `backoff_test.checkExpWithAct` for gocognit linter
-* Refactored `internal/xsql/dsn.go` and extract func `dsn.bindTablePathPrefixInConnectorOptions` for gocognit linter
-* Refactored `sugar/path.go` and extract funcs `path.removeWithPrefix`,  `path.removeEntry` for gocognit linter
-* Refactored `internal/stack/record.go` and extract func `record.findFileNameAndPkgPath` for gocognit linter
+## v3.57.1
+* Added logs over query service internals
+* Changed `trace.Query` events
+* Changed visibility of `query.{Do,DoTx}Options` from public to private
+
+## v3.57.0
+* Added experimental implementation of query service client
+* Fixed sometime panic on topic writer closing
+* Added experimental query parameters builder `ydb.ParamsBuilder()`
+* Changed types of `table/table.{QueryParameters,ParameterOption}` to aliases on `internal/params.{Parameters,NamedValue}`
+* Fixed bug with optional decimal serialization 
+
+## v3.56.2
+* Fixed return private error for commit to stopped partition in topic reader.
+* Stopped wrapping err error as transport error at topic streams (internals)
+
+## v3.56.1
+* Fixed fixenv usage (related to tests only)
+
+## v3.56.0
+* Fixed handle of operational errors in topic streams
+* The minimum version of Go in `ydb-go-sdk` has been raised to `go1.21`
 * Fixed topic writer infinite reconnections in some cases
 * Refactored nil on err `internal/grpcwrapper/rawydb/issues.go`, when golangci-lint nilerr enabled
 * Refactored nil on err `internal/grpcwrapper/rawtopic/describe_topic.go`, when golangci-lint nilerr enabled
 
+## v3.55.3
+* Fixed handle of operational errors in topic streams (backported fix only)
+
 ## v3.55.2
 * Fixed init info in topic writer, when autoseq num turned off.
 
@@ -44,14 +48,14 @@
 
 ## v3.54.2
 * Added context to some internal methods for better tracing
-* Added `trace.FunctionID` helper and `FunctionID` field to trace start info's 
+* Added `trace.FunctionID` helper and `FunctionID` field to trace start info's
 * Replaced lazy initialization of ydb clients (table, topic, etc.) to explicit initialization on `ydb.Open` step
 
 ## v3.54.1
-* Fixed inconsistent labels in `metrics` 
+* Fixed inconsistent labels in `metrics`
 
 ## v3.54.0
-* Allowed `sql.LevelSerializable` isolation level in read-write mode in `database/sql` transactions 
+* Allowed `sql.LevelSerializable` isolation level in read-write mode in `database/sql` transactions
 * Refactored traces and metrics
 * Added `{retry,table}.WithLabel` options for mark retriers calls
 * Added `ydb.WithTraceRetry` option
@@ -75,7 +79,7 @@
 * Fixed stringification of credentials object
 
 ## v3.53.2
-* Fixed panic when try to unwrap values with more than 127 columns with custom ydb unmarshaler 
+* Fixed panic when try to unwrap values with more than 127 columns with custom ydb unmarshaler
 
 ## v3.53.1
 * Bumps `github.com/ydb-platform/ydb-go-genproto` for support `query` service
@@ -224,7 +228,7 @@
 * Added `table/options.WithCallOptions` options for append custom grpc call options into `session.{BulkUpsert,Execute,StreamExecuteScanQuery}`
 * Supported fake transactions in `database/sql` driver over connector option `ydb.WithFakeTx(queryMode)` and connection string param `go_fake_tx`
 * Removed `testutil/timeutil` package (all usages replaced with `clockwork` package)
-* Changed behaviour of retryer on transport errors `cancelled` and `deadline exceeded` - will retry idempotent operation if context is not done  
+* Changed behaviour of retryer on transport errors `cancelled` and `deadline exceeded` - will retry idempotent operation if context is not done
 * Added address of node to operation error description as optional
 * Fixed bug with put session from unknown node
 * Fixed bug with parsing of `TzTimestamp` without microseconds

CONTRIBUTING.md

@@ -56,7 +56,7 @@ go test -race -tags fast ./...
 ##### All tests
 
 ```sh
-docker run -itd --name ydb -dp 2135:2135 -dp 2136:2136 -dp 8765:8765 -v `pwd`/ydb_certs:/ydb_certs -e YDB_LOCAL_SURVIVE_RESTART=true -e YDB_USE_IN_MEMORY_PDISKS=true -h localhost cr.yandex/yc/yandex-docker-local-ydb:latest
+docker run -itd --name ydb -dp 2135:2135 -dp 2136:2136 -dp 8765:8765 -v `pwd`/ydb_certs:/ydb_certs -e YDB_LOCAL_SURVIVE_RESTART=true -e YDB_USE_IN_MEMORY_PDISKS=true -h localhost ydbplatform/local-ydb:latest
 export YDB_CONNECTION_STRING="grpcs://localhost:2135/local"
 export YDB_SSL_ROOT_CERTIFICATES_FILE="`pwd`/ydb_certs/ca.pem"
 export YDB_SESSIONS_SHUTDOWN_URLS="http://localhost:8765/actors/kqp_proxy?force_shutdown=all"

connection.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/ydb-platform/ydb-go-sdk/v3/coordination"
 	"github.com/ydb-platform/ydb-go-sdk/v3/discovery"
+	"github.com/ydb-platform/ydb-go-sdk/v3/query"
 	"github.com/ydb-platform/ydb-go-sdk/v3/ratelimiter"
 	"github.com/ydb-platform/ydb-go-sdk/v3/scheme"
 	"github.com/ydb-platform/ydb-go-sdk/v3/scripting"
@@ -34,6 +35,13 @@ type Connection interface {
 	// Table returns table client
 	Table() table.Client
 
+	// Query returns query client
+	//
+	// # Experimental
+	//
+	// Notice: This API is EXPERIMENTAL and may be changed or removed in a later release.
+	Query() query.Client
+
 	// Scheme returns scheme client
 	Scheme() scheme.Client
 

driver.go

@@ -20,6 +20,8 @@ import (
 	discoveryConfig "github.com/ydb-platform/ydb-go-sdk/v3/internal/discovery/config"
 	"github.com/ydb-platform/ydb-go-sdk/v3/internal/dsn"
 	"github.com/ydb-platform/ydb-go-sdk/v3/internal/endpoint"
+	internalQuery "github.com/ydb-platform/ydb-go-sdk/v3/internal/query"
+	queryConfig "github.com/ydb-platform/ydb-go-sdk/v3/internal/query/config"
 	internalRatelimiter "github.com/ydb-platform/ydb-go-sdk/v3/internal/ratelimiter"
 	ratelimiterConfig "github.com/ydb-platform/ydb-go-sdk/v3/internal/ratelimiter/config"
 	internalScheme "github.com/ydb-platform/ydb-go-sdk/v3/internal/scheme"
@@ -35,6 +37,7 @@ import (
 	"github.com/ydb-platform/ydb-go-sdk/v3/internal/xsql"
 	"github.com/ydb-platform/ydb-go-sdk/v3/internal/xsync"
 	"github.com/ydb-platform/ydb-go-sdk/v3/log"
+	"github.com/ydb-platform/ydb-go-sdk/v3/query"
 	"github.com/ydb-platform/ydb-go-sdk/v3/ratelimiter"
 	"github.com/ydb-platform/ydb-go-sdk/v3/scheme"
 	"github.com/ydb-platform/ydb-go-sdk/v3/scripting"
@@ -47,7 +50,7 @@ import (
 var _ Connection = (*Driver)(nil)
 
 // Driver type provide access to YDB service clients
-type Driver struct { //nolint:maligned
+type Driver struct {
 	ctx       context.Context // cancel while Driver.Close called.
 	ctxCancel context.CancelFunc
 
@@ -68,6 +71,9 @@ type Driver struct { //nolint:maligned
 	table        *internalTable.Client
 	tableOptions []tableConfig.Option
 
+	query        *internalQuery.Client
+	queryOptions []queryConfig.Option
+
 	scripting        *internalScripting.Client
 	scriptingOptions []scriptingConfig.Option
 
@@ -141,6 +147,7 @@ func (d *Driver) Close(ctx context.Context) (finalErr error) {
 		d.scheme.Close,
 		d.scripting.Close,
 		d.table.Close,
+		d.query.Close,
 		d.topic.Close,
 		d.balancer.Close,
 		d.pool.Release,
@@ -180,6 +187,15 @@ func (d *Driver) Table() table.Client {
 	return d.table
 }
 
+// Query returns query client
+//
+// # Experimental
+//
+// Notice: This API is EXPERIMENTAL and may be changed or removed in a later release.
+func (d *Driver) Query() query.Client {
+	return d.query
+}
+
 // Scheme returns scheme client
 func (d *Driver) Scheme() scheme.Client {
 	return d.scheme
@@ -399,6 +415,22 @@ func (d *Driver) connect(ctx context.Context) (err error) {
 		return xerrors.WithStackTrace(err)
 	}
 
+	d.query, err = internalQuery.New(ctx,
+		d.balancer,
+		queryConfig.New(
+			append(
+				// prepend common params from root config
+				[]queryConfig.Option{
+					queryConfig.With(d.config.Common),
+				},
+				d.queryOptions...,
+			)...,
+		),
+	)
+	if err != nil {
+		return xerrors.WithStackTrace(err)
+	}
+
 	d.scheme, err = internalScheme.New(ctx,
 		d.balancer,
 		schemeConfig.New(