diff --git a/CHANGELOG.md b/CHANGELOG.md index 984b585..336a04d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Column type detection no longer declares a type the storage will not match. `strconv.ParseFloat` accepts Go source syntax that SQLite's numeric affinity does not convert, so a digit-separating underscore (`1_000`) or a hexadecimal float (`0x1p4`) made the column `REAL` while every value in it was stored as text. A datetime alongside a number did the same: the numeric type won on confidence and the datetime was stored as text under it. Both are TEXT now, which is what the storage always was. A caller reading the schema to plan a numeric comparison had no way to detect the difference. + ## [0.35.1] - 2026-08-06 ### Fixed diff --git a/types.go b/types.go index b6452ae..55a98cb 100644 --- a/types.go +++ b/types.go @@ -659,10 +659,31 @@ func isFloat(value string) bool { return false } + // strconv.ParseFloat accepts Go source syntax: digit-separating underscores + // ("1_000") and hexadecimal floats ("0x1p4"). SQLite's numeric affinity + // converts neither, so calling them numeric declared a REAL column whose + // values it then stored as text, leaving the schema and typeof() disagreeing. + if hasGoOnlyNumericSyntax(value) { + return false + } + _, err := strconv.ParseFloat(value, 64) return err == nil } +// hasGoOnlyNumericSyntax reports whether value uses numeric syntax that Go's +// parsers accept and SQLite's numeric affinity does not convert: an underscore +// separator, or the "0x" prefix that introduces a hexadecimal (and possibly +// p-exponent) literal. +func hasGoOnlyNumericSyntax(value string) bool { + digits := strings.TrimSpace(value) + digits = strings.TrimPrefix(strings.TrimPrefix(digits, "+"), "-") + if strings.Contains(digits, "_") { + return true + } + return len(digits) > 1 && digits[0] == '0' && (digits[1] == 'x' || digits[1] == 'X') +} + // isIntegerLiteralOverflowingInt64 reports whether value is an integer literal // (optional leading '+'/'-' followed solely by ASCII digits) whose magnitude // exceeds the range representable by int64. Such values can only be stored @@ -697,6 +718,15 @@ func selectColumnType(typeCounts map[columnType]int, totalCount int) columnType return columnTypeText } + // A datetime is stored as text, so a column that also holds a number has no + // type covering both, exactly as a column holding text does. Picking the + // numeric one declared INTEGER or REAL over values SQLite then stored as + // text, leaving the schema and typeof() disagreeing. + if typeCounts[columnTypeDatetime] > 0 && + typeCounts[columnTypeInteger]+typeCounts[columnTypeReal] > 0 { + return columnTypeText + } + // Calculate confidence for each type datetimeConfidence := float64(typeCounts[columnTypeDatetime]) / float64(totalCount) realConfidence := float64(typeCounts[columnTypeReal]) / float64(totalCount) diff --git a/types_test.go b/types_test.go index 260e081..9329bd2 100644 --- a/types_test.go +++ b/types_test.go @@ -712,6 +712,17 @@ func TestIsFloat(t *testing.T) { {"no digits", "abc", false}, {"multiple dots", "12.34.56", false}, {"invalid scientific", "1e", false}, + + // Go's parser accepts these spellings and SQLite's numeric affinity does + // not convert them. Calling them numeric declared a REAL column that + // stored every value as text, so the schema and typeof() disagreed. + {"underscore separators", "1_000", false}, + {"underscore in a decimal", "1_000.5", false}, + {"short underscore form", "1_0", false}, + {"hexadecimal float", "0x1p4", false}, + {"hexadecimal integer", "0x10", false}, + {"binary literal", "0b101", false}, + {"octal literal", "0o17", false}, } for _, tt := range tests { @@ -756,6 +767,31 @@ func TestSelectColumnType(t *testing.T) { totalCount: 10, expected: columnTypeText, }, + { + // A datetime is stored as text, so a column holding one alongside a + // number has no type that covers both. Answering INTEGER declared a + // schema the storage did not match. + name: "datetime mixed with integer falls back to text", + typeCounts: map[columnType]int{ + columnTypeInteger: 1, + columnTypeReal: 0, + columnTypeDatetime: 1, + columnTypeText: 0, + }, + totalCount: 2, + expected: columnTypeText, + }, + { + name: "datetime mixed with real falls back to text", + typeCounts: map[columnType]int{ + columnTypeInteger: 0, + columnTypeReal: 3, + columnTypeDatetime: 1, + columnTypeText: 0, + }, + totalCount: 4, + expected: columnTypeText, + }, { name: "high confidence datetime", typeCounts: map[columnType]int{ @@ -769,6 +805,19 @@ func TestSelectColumnType(t *testing.T) { }, { name: "low confidence fallback to most common", + typeCounts: map[columnType]int{ + columnTypeInteger: 3, + columnTypeReal: 4, + columnTypeDatetime: 0, + columnTypeText: 0, + }, + totalCount: 7, + expected: columnTypeReal, + }, + { + // REAL used to win this on confidence, and the two datetimes in it + // were then stored as text under a REAL declaration. + name: "low confidence numerics with a datetime fall back to text", typeCounts: map[columnType]int{ columnTypeInteger: 3, columnTypeReal: 4, @@ -776,7 +825,7 @@ func TestSelectColumnType(t *testing.T) { columnTypeText: 0, }, totalCount: 10, - expected: columnTypeReal, + expected: columnTypeText, }, }