From 1c34cfa0841abe897d109ead080dd0757a623c1d Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 5 May 2026 22:17:07 +0900 Subject: [PATCH 1/2] fix(ffitemplate): allow non-string options pre-database-init --- ffitemplate/_tmpl/driver.go.tmpl | 80 +++++++++++++++++++++++++------- ffitemplate/main.go | 9 +++- 2 files changed, 71 insertions(+), 18 deletions(-) diff --git a/ffitemplate/_tmpl/driver.go.tmpl b/ffitemplate/_tmpl/driver.go.tmpl index ccc5703..e0a72bd 100644 --- a/ffitemplate/_tmpl/driver.go.tmpl +++ b/ffitemplate/_tmpl/driver.go.tmpl @@ -468,10 +468,17 @@ func exportRecordReader(rdr array.RecordReader, stream *C.struct_ArrowArrayStrea rdr.Retain() } +type unappliedOpt struct { + stringVal *string + int64Val *int64 + byteVal []byte + doubleVal *float64 +} + type cDatabase struct { cancellableContext - opts map[string]string + opts map[string]unappliedOpt db driverbase.Database } @@ -576,12 +583,35 @@ func {{.Prefix}}DatabaseInit(db *C.struct_AdbcDatabase, err *C.struct_AdbcError) return C.ADBC_STATUS_INVALID_STATE } - adb, aerr := drv.NewDatabaseWithContext(cdb.newContext(), cdb.opts) + stringOpts := map[string]string{} + for k, v := range cdb.opts { + if v.stringVal != nil { + stringOpts[k] = *v.stringVal + } + } + ctx := cdb.newContext() + adb, aerr := drv.NewDatabaseWithContext(ctx, stringOpts) if aerr != nil { return C.AdbcStatusCode(errToAdbcErr(err, aerr)) } cdb.db = adb.(driverbase.Database) + for k, v := range cdb.opts { + switch { + case v.stringVal != nil: + continue + case v.int64Val != nil: + aerr = cdb.db.SetOptionInt(ctx, k, *v.int64Val) + case v.byteVal != nil: + aerr = cdb.db.SetOptionBytes(ctx, k, v.byteVal) + case v.doubleVal != nil: + aerr = cdb.db.SetOptionDouble(ctx, k, *v.doubleVal) + } + if aerr != nil { + return C.AdbcStatusCode(errToAdbcErr(err, aerr)) + } + } + initLoggingFromEnv(cdb.db) return C.ADBC_STATUS_OK } @@ -601,7 +631,7 @@ func {{.Prefix}}DatabaseNew(db *C.struct_AdbcDatabase, err *C.struct_AdbcError) setErr(err, "AdbcDatabaseNew: database already allocated") return C.ADBC_STATUS_INVALID_STATE } - dbobj := &cDatabase{opts: make(map[string]string)} + dbobj := &cDatabase{opts: make(map[string]unappliedOpt)} hndl := cgo.NewHandle(dbobj) db.private_data = createHandle(hndl) return C.ADBC_STATUS_OK @@ -655,7 +685,7 @@ func {{.Prefix}}DatabaseSetOption(db *C.struct_AdbcDatabase, key, value *C.cchar e := cdb.db.SetOption(cdb.newContext(), k, v) return C.AdbcStatusCode(errToAdbcErr(err, e)) } else { - cdb.opts[k] = v + cdb.opts[k] = unappliedOpt { stringVal: new(v) } } return C.ADBC_STATUS_OK @@ -668,13 +698,19 @@ func {{.Prefix}}DatabaseSetOptionBytes(db *C.struct_AdbcDatabase, key *C.cchar_t code = poison(err, "AdbcDatabaseSetOptionBytes", e) } }() - cdb := checkDBInit(db, err, "AdbcDatabaseSetOptionBytes") - if cdb == nil { + if !checkDBAlloc(db, err, "AdbcDatabaseSetOptionBytes") { return C.ADBC_STATUS_INVALID_STATE } + cdb := getFromHandle[cDatabase](db.private_data) + k := C.GoString(key) + v := fromCArr[byte](value, int(length)) - e := cdb.db.SetOptionBytes(cdb.newContext(), C.GoString(key), fromCArr[byte](value, int(length))) - return C.AdbcStatusCode(errToAdbcErr(err, e)) + if cdb.db != nil { + e := cdb.db.SetOptionBytes(cdb.newContext(), k, v) + return C.AdbcStatusCode(errToAdbcErr(err, e)) + } + cdb.opts[k] = unappliedOpt { byteVal: v } + return C.ADBC_STATUS_OK } //export {{.Prefix}}DatabaseSetOptionDouble @@ -684,13 +720,19 @@ func {{.Prefix}}DatabaseSetOptionDouble(db *C.struct_AdbcDatabase, key *C.cchar_ code = poison(err, "AdbcDatabaseSetOptionDouble", e) } }() - cdb := checkDBInit(db, err, "AdbcDatabaseSetOptionDouble") - if cdb == nil { + if !checkDBAlloc(db, err, "AdbcDatabaseSetOptionDouble") { return C.ADBC_STATUS_INVALID_STATE } + cdb := getFromHandle[cDatabase](db.private_data) + k := C.GoString(key) + v := float64(value) - e := cdb.db.SetOptionDouble(cdb.newContext(), C.GoString(key), float64(value)) - return C.AdbcStatusCode(errToAdbcErr(err, e)) + if cdb.db != nil { + e := cdb.db.SetOptionDouble(cdb.newContext(), k, v) + return C.AdbcStatusCode(errToAdbcErr(err, e)) + } + cdb.opts[k] = unappliedOpt { doubleVal: new(v) } + return C.ADBC_STATUS_OK } //export {{.Prefix}}DatabaseSetOptionInt @@ -700,13 +742,19 @@ func {{.Prefix}}DatabaseSetOptionInt(db *C.struct_AdbcDatabase, key *C.cchar_t, code = poison(err, "AdbcDatabaseSetOptionInt", e) } }() - cdb := checkDBInit(db, err, "AdbcDatabaseSetOptionInt") - if cdb == nil { + if !checkDBAlloc(db, err, "AdbcDatabaseSetOptionInt") { return C.ADBC_STATUS_INVALID_STATE } + cdb := getFromHandle[cDatabase](db.private_data) + k := C.GoString(key) + v := int64(value) - e := cdb.db.SetOptionInt(cdb.newContext(), C.GoString(key), int64(value)) - return C.AdbcStatusCode(errToAdbcErr(err, e)) + if cdb.db != nil { + e := cdb.db.SetOptionInt(cdb.newContext(), k, v) + return C.AdbcStatusCode(errToAdbcErr(err, e)) + } + cdb.opts[k] = unappliedOpt { int64Val: new(v) } + return C.ADBC_STATUS_OK } type cConn struct { diff --git a/ffitemplate/main.go b/ffitemplate/main.go index 6d9498a..d89eae4 100644 --- a/ffitemplate/main.go +++ b/ffitemplate/main.go @@ -187,9 +187,14 @@ func process(data any, specs []pathSpec) { } if f != nil { - generated, err = f(generated) + orig := generated + generated, err = f(orig) if err != nil { - log.Fatalf("error formatting '%s': %s", spec.in, err) + // write bad file for debugging + if err := os.WriteFile(spec.out, orig, fileMode(spec.in)); err != nil { + log.Printf("could not write unformatted '%s': %s", spec.out, err) + } + log.Fatalf("error formatting '%s': %s", spec.out, err) } } if err := os.WriteFile(spec.out, generated, fileMode(spec.in)); err != nil { From 5f956bd6b9f1f22cb789a0eeaa44da22f39b312a Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 19 May 2026 13:30:16 +0900 Subject: [PATCH 2/2] [WIP] test behavior of RecordFromJSON --- testutil/foo.go | 24 ++++++++++++++++++++ testutil/go.mod | 8 ++++++- testutil/go.sum | 2 ++ testutil/testutil.go | 4 ++-- testutil/testutil_test.go | 46 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 testutil/foo.go create mode 100644 testutil/testutil_test.go diff --git a/testutil/foo.go b/testutil/foo.go new file mode 100644 index 0000000..9b07a82 --- /dev/null +++ b/testutil/foo.go @@ -0,0 +1,24 @@ +package main + +import ( + "bytes" + "fmt" + "log" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" +) + +func main() { + mem := memory.DefaultAllocator + schema := arrow.NewSchema([]arrow.Field{ + {Name: "a", Type: arrow.FixedWidthTypes.Duration_s}, + }, nil) + json := `[{"a": -9223372036854775808}, {"a": 9223372036854775807}]` + record, _, err := array.RecordFromJSON(mem, schema, bytes.NewReader([]byte(json)), array.WithUseNumber()) + if err != nil { + log.Fatal(err) + } + fmt.Println(record) +} diff --git a/testutil/go.mod b/testutil/go.mod index 55b03f9..43d9bde 100644 --- a/testutil/go.mod +++ b/testutil/go.mod @@ -16,13 +16,19 @@ module github.com/adbc-drivers/driverbase-go/testutil go 1.26.0 -require github.com/apache/arrow-go/v18 v18.6.0 +require ( + github.com/apache/arrow-go/v18 v18.6.0 + github.com/stretchr/testify v1.11.1 +) require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/sys v0.43.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/testutil/go.sum b/testutil/go.sum index fc8d0e1..06c5f18 100644 --- a/testutil/go.sum +++ b/testutil/go.sum @@ -32,5 +32,7 @@ golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/testutil/testutil.go b/testutil/testutil.go index cbe6743..cb33aad 100644 --- a/testutil/testutil.go +++ b/testutil/testutil.go @@ -48,7 +48,7 @@ func CheckedCloseWithContext(t *testing.T, obj CloserWithContext, ctx context.Co // ArrayFromJSON is the same as array.FromJSON, but fails the test on error. func ArrayFromJSON(t *testing.T, mem memory.Allocator, dt arrow.DataType, json string) arrow.Array { - record, _, err := array.FromJSON(mem, dt, bytes.NewReader([]byte(json))) + record, _, err := array.FromJSON(mem, dt, bytes.NewReader([]byte(json)), array.WithUseNumber()) if err != nil { t.Fatalf("failed to create array from JSON: %v", err) } @@ -57,7 +57,7 @@ func ArrayFromJSON(t *testing.T, mem memory.Allocator, dt arrow.DataType, json s // RecordFromJSON is the same as array.RecordFromJSON, but fails the test on error. func RecordFromJSON(t *testing.T, mem memory.Allocator, schema *arrow.Schema, json string) arrow.RecordBatch { - record, _, err := array.RecordFromJSON(mem, schema, bytes.NewReader([]byte(json))) + record, _, err := array.RecordFromJSON(mem, schema, bytes.NewReader([]byte(json)), array.WithUseNumber()) if err != nil { t.Fatalf("failed to create record from JSON: %v", err) } diff --git a/testutil/testutil_test.go b/testutil/testutil_test.go new file mode 100644 index 0000000..6a990e6 --- /dev/null +++ b/testutil/testutil_test.go @@ -0,0 +1,46 @@ +// Copyright (c) 2025 ADBC Drivers Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testutil_test + +import ( + "testing" + + "github.com/adbc-drivers/driverbase-go/testutil" + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/stretchr/testify/assert" +) + +func TestArrayFromJSON(t *testing.T) { + mem := memory.DefaultAllocator + dt := arrow.FixedWidthTypes.Duration_s + arr := testutil.ArrayFromJSON(t, mem, dt, `[-9223372036854775808, 9223372036854775807]`).(*array.Duration) + assert.Equal(t, arrow.Duration(-9223372036854775808), arr.Value(0)) + assert.Equal(t, arrow.Duration(9223372036854775807), arr.Value(1)) +} + +func TestRecordFromJSON(t *testing.T) { + mem := memory.DefaultAllocator + schema := arrow.NewSchema([]arrow.Field{ + {Name: "a", Type: arrow.FixedWidthTypes.Duration_s}, + }, nil) + json := `[{"a": -9223372036854775808}, {"a": 9223372036854775807}]` + record := testutil.RecordFromJSON(t, mem, schema, json) + arr := record.Column(0).(*array.Duration) + t.Logf("%v", arr) + assert.Equal(t, arrow.Duration(-9223372036854775808), arr.Value(0)) + assert.Equal(t, arrow.Duration(9223372036854775807), arr.Value(1)) +}