Skip to content

Commit b290658

Browse files
charlieviethcharlievieth
committed
Fix exponential memory allocation in Exec and improve performance
This commit changes SQLiteConn.Exec to use the raw Go query string instead of repeatedly converting it to a C string (which it would do for every statement in the provided query). This yields a ~20% performance improvement for a query containing one statement and a significantly larger improvement when the query contains multiple statements as is common when importing a SQL dump (our benchmark shows a 5x improvement for handling 1k SQL statements). Additionally, this commit improves the performance of Exec by 2x or more and makes number and size of allocations constant when there are no bind parameters (the performance improvement scales with the number of SQL statements in the query). This is achieved by having the entire query processed in C code thus requiring only one CGO call. The speedup for Exec'ing single statement queries means that wrapping simple statements in a transaction is now twice as fast. This commit also improves the test coverage of Exec, which previously failed to test that Exec could process multiple statements like INSERT. It also adds some Exec specific benchmarks that highlight both the improvements here and the overhead of using a cancellable Context. This commit is a slimmed down and improved version of PR mattn#1133: mattn#1133 ``` goos: darwin goarch: arm64 pkg: github.com/mattn/go-sqlite3 cpu: Apple M1 Max │ b.txt │ n.txt │ │ sec/op │ sec/op vs base │ Suite/BenchmarkExec/Params-10 1.434µ ± 1% 1.186µ ± 0% -17.27% (p=0.000 n=10) Suite/BenchmarkExec/NoParams-10 1267.5n ± 0% 759.2n ± 1% -40.10% (p=0.000 n=10) Suite/BenchmarkExecContext/Params-10 2.886µ ± 0% 2.517µ ± 0% -12.80% (p=0.000 n=10) Suite/BenchmarkExecContext/NoParams-10 2.605µ ± 1% 1.829µ ± 1% -29.81% (p=0.000 n=10) Suite/BenchmarkExecStep-10 1852.6µ ± 1% 582.3µ ± 0% -68.57% (p=0.000 n=10) Suite/BenchmarkExecContextStep-10 3053.3µ ± 3% 582.0µ ± 0% -80.94% (p=0.000 n=10) Suite/BenchmarkExecTx-10 4.126µ ± 2% 2.200µ ± 1% -46.67% (p=0.000 n=10) geomean 16.40µ 8.455µ -48.44% │ b.txt │ n.txt │ │ B/op │ B/op vs base │ Suite/BenchmarkExec/Params-10 248.0 ± 0% 240.0 ± 0% -3.23% (p=0.000 n=10) Suite/BenchmarkExec/NoParams-10 128.00 ± 0% 64.00 ± 0% -50.00% (p=0.000 n=10) Suite/BenchmarkExecContext/Params-10 408.0 ± 0% 400.0 ± 0% -1.96% (p=0.000 n=10) Suite/BenchmarkExecContext/NoParams-10 288.0 ± 0% 208.0 ± 0% -27.78% (p=0.000 n=10) Suite/BenchmarkExecStep-10 5406674.50 ± 0% 64.00 ± 0% -100.00% (p=0.000 n=10) Suite/BenchmarkExecContextStep-10 5566758.5 ± 0% 208.0 ± 0% -100.00% (p=0.000 n=10) Suite/BenchmarkExecTx-10 712.0 ± 0% 520.0 ± 0% -26.97% (p=0.000 n=10) geomean 4.899Ki 189.7 -96.22% │ b.txt │ n.txt │ │ allocs/op │ allocs/op vs base │ Suite/BenchmarkExec/Params-10 10.000 ± 0% 9.000 ± 0% -10.00% (p=0.000 n=10) Suite/BenchmarkExec/NoParams-10 7.000 ± 0% 4.000 ± 0% -42.86% (p=0.000 n=10) Suite/BenchmarkExecContext/Params-10 12.00 ± 0% 11.00 ± 0% -8.33% (p=0.000 n=10) Suite/BenchmarkExecContext/NoParams-10 9.000 ± 0% 6.000 ± 0% -33.33% (p=0.000 n=10) Suite/BenchmarkExecStep-10 7000.000 ± 0% 4.000 ± 0% -99.94% (p=0.000 n=10) Suite/BenchmarkExecContextStep-10 9001.000 ± 0% 6.000 ± 0% -99.93% (p=0.000 n=10) Suite/BenchmarkExecTx-10 27.00 ± 0% 18.00 ± 0% -33.33% (p=0.000 n=10) geomean 74.60 7.224 -90.32% ```
1 parent cf831bd commit b290658

File tree

3 files changed

+190
-22
lines changed

3 files changed

+190
-22
lines changed

‎sqlite3.go

+150-22
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,59 @@ _sqlite3_prepare_v2_internal(sqlite3 *db, const char *zSql, int nBytes, sqlite3_
137137
}
138138
#endif
139139
140+
static int _sqlite3_prepare_v2(sqlite3 *db, const char *zSql, int nBytes, sqlite3_stmt **ppStmt, int *oBytes) {
141+
const char *tail = NULL;
142+
int rv = _sqlite3_prepare_v2_internal(db, zSql, nBytes, ppStmt, &tail);
143+
if (rv != SQLITE_OK) {
144+
return rv;
145+
}
146+
if (tail == NULL) {
147+
return rv; // NB: this should not happen
148+
}
149+
// Set oBytes to the number of bytes consumed instead of using the **pzTail
150+
// out param since that requires storing a Go pointer in a C pointer, which
151+
// is not allowed by CGO and will cause runtime.cgoCheckPointer to fail.
152+
*oBytes = tail - zSql;
153+
return rv;
154+
}
155+
156+
// _sqlite3_exec_no_args executes all of the statements in sql. None of the
157+
// statements are allowed to have positional arguments.
158+
int _sqlite3_exec_no_args(sqlite3 *db, const char *sql, int nbytes,
159+
int64_t *rowid, int64_t *changes) {
160+
if (nbytes < 0) {
161+
nbytes = strlen(sql);
162+
}
163+
sqlite3_stmt *stmt;
164+
const char *tail = NULL;
165+
while (*sql && nbytes > 0) {
166+
stmt = NULL;
167+
int rv = sqlite3_prepare_v2(db, sql, nbytes, &stmt, &tail);
168+
if (rv != SQLITE_OK) {
169+
return rv;
170+
}
171+
172+
do {
173+
rv = sqlite3_step(stmt);
174+
} while (rv == SQLITE_ROW);
175+
176+
*rowid = sqlite3_last_insert_rowid(db);
177+
*changes = sqlite3_changes64(db);
178+
179+
if (rv != SQLITE_OK && rv != SQLITE_DONE) {
180+
sqlite3_finalize(stmt);
181+
return rv;
182+
}
183+
rv = sqlite3_finalize(stmt);
184+
if (rv != SQLITE_OK) {
185+
return rv;
186+
}
187+
nbytes -= tail - sql;
188+
sql = tail;
189+
}
190+
return SQLITE_OK;
191+
}
192+
140193
void _sqlite3_result_text(sqlite3_context* ctx, const char* s) {
141194
sqlite3_result_text(ctx, s, -1, &free);
142195
}
@@ -858,54 +911,122 @@ func (c *SQLiteConn) Exec(query string, args []driver.Value) (driver.Result, err
858911
}
859912

860913
func (c *SQLiteConn) exec(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
861-
start := 0
914+
// Trim the query. This is mostly important for getting rid
915+
// of any trailing space.
916+
query = strings.TrimSpace(query)
917+
if len(args) > 0 {
918+
return c.execArgs(ctx, query, args)
919+
}
920+
return c.execNoArgs(ctx, query)
921+
}
922+
923+
func (c *SQLiteConn) execArgs(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
924+
var (
925+
stmtArgs []driver.NamedValue
926+
start int
927+
s SQLiteStmt // escapes to the heap so reuse it
928+
sz C.int // number of query bytes consumed: escapes to the heap
929+
)
862930
for {
863-
s, err := c.prepare(ctx, query)
864-
if err != nil {
865-
return nil, err
931+
s = SQLiteStmt{c: c} // reset
932+
sz = 0
933+
rv := C._sqlite3_prepare_v2(c.db, (*C.char)(unsafe.Pointer(stringData(query))),
934+
C.int(len(query)), &s.s, &sz)
935+
if rv != C.SQLITE_OK {
936+
return nil, c.lastError()
866937
}
938+
query = strings.TrimSpace(query[sz:])
939+
867940
var res driver.Result
868-
if s.(*SQLiteStmt).s != nil {
869-
stmtArgs := make([]driver.NamedValue, 0, len(args))
941+
if s.s != nil {
870942
na := s.NumInput()
871943
if len(args)-start < na {
872-
s.Close()
944+
s.finalize()
873945
return nil, fmt.Errorf("not enough args to execute query: want %d got %d", na, len(args))
874946
}
875947
// consume the number of arguments used in the current
876948
// statement and append all named arguments not
877949
// contained therein
878-
if len(args[start:start+na]) > 0 {
879-
stmtArgs = append(stmtArgs, args[start:start+na]...)
880-
for i := range args {
881-
if (i < start || i >= na) && args[i].Name != "" {
882-
stmtArgs = append(stmtArgs, args[i])
883-
}
884-
}
885-
for i := range stmtArgs {
886-
stmtArgs[i].Ordinal = i + 1
950+
if stmtArgs == nil {
951+
stmtArgs = make([]driver.NamedValue, 0, na)
952+
}
953+
stmtArgs = append(stmtArgs[:0], args[start:start+na]...)
954+
for i := range args {
955+
if (i < start || i >= na) && args[i].Name != "" {
956+
stmtArgs = append(stmtArgs, args[i])
887957
}
888958
}
889-
res, err = s.(*SQLiteStmt).exec(ctx, stmtArgs)
959+
for i := range stmtArgs {
960+
stmtArgs[i].Ordinal = i + 1
961+
}
962+
var err error
963+
res, err = s.exec(ctx, stmtArgs)
890964
if err != nil && err != driver.ErrSkip {
891-
s.Close()
965+
s.finalize()
892966
return nil, err
893967
}
894968
start += na
895969
}
896-
tail := s.(*SQLiteStmt).t
897-
s.Close()
898-
if tail == "" {
970+
s.finalize()
971+
if len(query) == 0 {
899972
if res == nil {
900973
// https://github.com/mattn/go-sqlite3/issues/963
901974
res = &SQLiteResult{0, 0}
902975
}
903976
return res, nil
904977
}
905-
query = tail
906978
}
907979
}
908980

981+
// execNoArgsSync processes every SQL statement in query. All processing occurs
982+
// in C code, which reduces the overhead of CGO calls.
983+
func (c *SQLiteConn) execNoArgsSync(query string) (_ driver.Result, err error) {
984+
var rowid, changes C.int64_t
985+
rv := C._sqlite3_exec_no_args(c.db, (*C.char)(unsafe.Pointer(stringData(query))),
986+
C.int(len(query)), &rowid, &changes)
987+
if rv != C.SQLITE_OK {
988+
err = c.lastError()
989+
}
990+
return &SQLiteResult{id: int64(rowid), changes: int64(changes)}, err
991+
}
992+
993+
func (c *SQLiteConn) execNoArgs(ctx context.Context, query string) (driver.Result, error) {
994+
done := ctx.Done()
995+
if done == nil {
996+
return c.execNoArgsSync(query)
997+
}
998+
999+
// Fast check if the Context is cancelled
1000+
if err := ctx.Err(); err != nil {
1001+
return nil, err
1002+
}
1003+
1004+
ch := make(chan struct{})
1005+
defer close(ch)
1006+
go func() {
1007+
select {
1008+
case <-done:
1009+
C.sqlite3_interrupt(c.db)
1010+
// Wait until signaled. We need to ensure that this goroutine
1011+
// will not call interrupt after this method returns, which is
1012+
// why we can't check if only done is closed when waiting below.
1013+
<-ch
1014+
case <-ch:
1015+
}
1016+
}()
1017+
1018+
res, err := c.execNoArgsSync(query)
1019+
1020+
// Stop the goroutine and make sure we're at a point where
1021+
// sqlite3_interrupt cannot be called again.
1022+
ch <- struct{}{}
1023+
1024+
if isInterruptErr(err) {
1025+
err = ctx.Err()
1026+
}
1027+
return res, err
1028+
}
1029+
9091030
// Query implements Queryer.
9101031
func (c *SQLiteConn) Query(query string, args []driver.Value) (driver.Rows, error) {
9111032
list := make([]driver.NamedValue, len(args))
@@ -1914,6 +2035,13 @@ func (s *SQLiteStmt) Close() error {
19142035
return nil
19152036
}
19162037

2038+
func (s *SQLiteStmt) finalize() {
2039+
if s.s != nil {
2040+
C.sqlite3_finalize(s.s)
2041+
s.s = nil
2042+
}
2043+
}
2044+
19172045
// NumInput return a number of parameters.
19182046
func (s *SQLiteStmt) NumInput() int {
19192047
return int(C.sqlite3_bind_parameter_count(s.s))

‎unsafe_go120.go

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
//go:build !go1.21
2+
// +build !go1.21
3+
4+
package sqlite3
5+
6+
import "unsafe"
7+
8+
// stringData is a safe version of unsafe.StringData that handles empty strings.
9+
func stringData(s string) *byte {
10+
if len(s) != 0 {
11+
b := *(*[]byte)(unsafe.Pointer(&s))
12+
return &b[0]
13+
}
14+
// The return value of unsafe.StringData
15+
// is unspecified if the string is empty.
16+
return &placeHolder[0]
17+
}

‎unsafe_go121.go

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
//go:build go1.21
2+
// +build go1.21
3+
4+
// The unsafe.StringData function was made available in Go 1.20 but it
5+
// was not until Go 1.21 that Go was changed to interpret the Go version
6+
// in go.mod (1.19 as of writing this) as the minimum version required
7+
// instead of the exact version.
8+
//
9+
// See: https://github.com/golang/go/issues/59033
10+
11+
package sqlite3
12+
13+
import "unsafe"
14+
15+
// stringData is a safe version of unsafe.StringData that handles empty strings.
16+
func stringData(s string) *byte {
17+
if len(s) != 0 {
18+
return unsafe.StringData(s)
19+
}
20+
// The return value of unsafe.StringData
21+
// is unspecified if the string is empty.
22+
return &placeHolder[0]
23+
}

0 commit comments

Comments
 (0)